From 14b4c56e5796e995706fec5b5124181a0f60bc35 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Wed, 28 Oct 2026 19:18:27 +0000 Subject: [PATCH] feat(chord,durable): Chord v2 slice 6b, Durable and jobs on the store StoreStorage (Durable's storage on Chord v2's store, @earendil-works/pi-durable/storage/store) passes Durable's storage conformance suite and runs Durable's harness tests (npm run test:store). The store gets owner names and claims, one id allocator, Durable's Storage semantics for refused steps, history none/latest/rewindable with reads at a point, archive revisions and applyBatch. self.follow follows Durable's synchronous observers as Chord sources. Persisted jobs: records and state in the owner's store, restarts as Durable's tasks, abort stored, deferred state. Chord's ./next export returns for the adapter surface. --- packages/chord/CHANGELOG.md | 1 + packages/chord/docs/next/benchmarks.md | 93 ++ packages/chord/docs/next/slices.md | 19 +- packages/chord/package.json | 5 + .../chord/src/next/documents/definition.ts | 9 + .../chord/src/next/documents/documents.ts | 18 +- packages/chord/src/next/documents/index.ts | 2 +- packages/chord/src/next/documents/internal.ts | 10 +- .../chord/src/next/documents/keyed-slot.ts | 22 +- .../chord/src/next/documents/persistence.ts | 67 +- packages/chord/src/next/documents/slot.ts | 38 +- .../chord/src/next/documents/transaction.ts | 65 +- packages/chord/src/next/host/attempt.ts | 85 +- packages/chord/src/next/host/follow.ts | 248 ++++ packages/chord/src/next/host/host.ts | 10 +- packages/chord/src/next/host/index.ts | 1 + packages/chord/src/next/host/object.ts | 70 +- packages/chord/src/next/host/routines.ts | 1 + packages/chord/src/next/index.ts | 50 + packages/chord/src/next/jobs/README.md | 14 +- packages/chord/src/next/jobs/definition.ts | 68 +- packages/chord/src/next/jobs/index.ts | 2 +- packages/chord/src/next/jobs/job.ts | 174 ++- packages/chord/src/next/jobs/persisted.ts | 100 ++ packages/chord/src/next/jobs/records.ts | 524 ++++++-- packages/chord/src/next/jobs/run-states.ts | 243 ++++ packages/chord/src/next/jobs/run.ts | 141 ++- packages/chord/src/next/jobs/running.ts | 3 +- packages/chord/src/next/jobs/stored.ts | 106 ++ packages/chord/src/next/remote/hold.ts | 1 + packages/chord/src/next/state/batch.ts | 27 + packages/chord/src/next/state/watchable.ts | 17 + packages/chord/src/next/store/README.md | 12 + packages/chord/src/next/store/adapter.ts | 152 +++ packages/chord/src/next/store/allocator.ts | 33 + packages/chord/src/next/store/applier.ts | 297 ++++- packages/chord/src/next/store/apply-batch.ts | 99 ++ packages/chord/src/next/store/archive.ts | 168 ++- packages/chord/src/next/store/claims.ts | 57 + packages/chord/src/next/store/engine.ts | 462 +++++-- packages/chord/src/next/store/export.ts | 39 +- packages/chord/src/next/store/import.ts | 105 +- packages/chord/src/next/store/outbox.ts | 17 +- packages/chord/src/next/store/refusal.ts | 43 + packages/chord/src/next/store/schema.ts | 66 +- packages/chord/src/next/store/sql.ts | 30 +- packages/chord/src/next/store/step.ts | 45 +- packages/chord/src/next/store/store.ts | 296 ++++- packages/chord/src/next/store/table-writes.ts | 12 +- packages/chord/test/next/bench/jobs.ts | 164 +++ packages/chord/test/next/bench/store.ts | 8 +- .../test/next/documents/persistence.test.ts | 34 +- .../test/next/documents/transaction.test.ts | 30 +- packages/chord/test/next/examples/19-jobs.ts | 2 +- .../test/next/examples/28-persisted-jobs.ts | 125 ++ packages/chord/test/next/host/attempt.test.ts | 4 +- packages/chord/test/next/host/follow.test.ts | 345 ++++++ packages/chord/test/next/host/host.test.ts | 17 +- .../test/next/host/member-source.test.ts | 1 + .../chord/test/next/jobs/definition.test.ts | 31 +- packages/chord/test/next/jobs/job.test.ts | 20 +- .../chord/test/next/jobs/persisted.test.ts | 1066 +++++++++++++++++ packages/chord/test/next/jobs/records.test.ts | 4 +- packages/chord/test/next/jobs/run.test.ts | 18 +- .../chord/test/next/state/watchable.test.ts | 1 + .../chord/test/next/store/allocator.test.ts | 24 + .../chord/test/next/store/applier.test.ts | 20 +- .../chord/test/next/store/apply-batch.test.ts | 290 +++++ .../chord/test/next/store/catalog.test.ts | 12 +- packages/chord/test/next/store/claims.test.ts | 72 ++ packages/chord/test/next/store/engine.test.ts | 181 ++- packages/chord/test/next/store/export.test.ts | 85 +- packages/chord/test/next/store/import.test.ts | 92 +- packages/chord/test/next/store/jsonl.test.ts | 4 +- packages/chord/test/next/store/outbox.test.ts | 9 +- packages/chord/test/next/store/query.test.ts | 28 +- .../chord/test/next/store/refusal.test.ts | 65 + packages/chord/test/next/store/sql.test.ts | 2 +- packages/chord/test/next/store/store.test.ts | 191 ++- .../test/next/store/table-changes.test.ts | 2 +- .../test/next/store/table-handle.test.ts | 8 +- .../test/next/store/table-migration.test.ts | 2 +- .../test/next/store/table-writes.test.ts | 9 +- packages/chord/test/next/support.ts | 64 +- packages/chord/test/sqlite/cloudflare.test.ts | 74 ++ packages/chord/test/sqlite/node.test.ts | 47 +- packages/durable/CHANGELOG.md | 1 + packages/durable/README.md | 2 + packages/durable/package.json | 9 +- packages/durable/src/storage/store/commit.ts | 327 +++++ .../durable/src/storage/store/documents.ts | 143 +++ packages/durable/src/storage/store/index.ts | 1 + packages/durable/src/storage/store/queries.ts | 137 +++ packages/durable/src/storage/store/storage.ts | 557 +++++++++ packages/durable/src/storage/store/tables.ts | 122 ++ .../src/testing/storage-conformance.ts | 24 + .../durable/test/conversation-view-bench.ts | 10 +- .../test/examples/35-durable-on-the-store.ts | 108 ++ .../durable/test/examples/36-durable-views.ts | 91 ++ packages/durable/test/storage-memory.ts | 11 +- packages/durable/test/storage.bench.ts | 15 +- packages/durable/test/store-storage-node.ts | 23 + .../durable/test/store-storage-support.ts | 46 + packages/durable/test/store-storage.test.ts | 592 +++++++++ packages/durable/vitest.store.config.ts | 47 + tsconfig.json | 1 + 106 files changed, 8714 insertions(+), 871 deletions(-) create mode 100644 packages/chord/src/next/host/follow.ts create mode 100644 packages/chord/src/next/index.ts create mode 100644 packages/chord/src/next/jobs/persisted.ts create mode 100644 packages/chord/src/next/jobs/run-states.ts create mode 100644 packages/chord/src/next/jobs/stored.ts create mode 100644 packages/chord/src/next/store/adapter.ts create mode 100644 packages/chord/src/next/store/allocator.ts create mode 100644 packages/chord/src/next/store/apply-batch.ts create mode 100644 packages/chord/src/next/store/claims.ts create mode 100644 packages/chord/src/next/store/refusal.ts create mode 100644 packages/chord/test/next/bench/jobs.ts create mode 100644 packages/chord/test/next/examples/28-persisted-jobs.ts create mode 100644 packages/chord/test/next/host/follow.test.ts create mode 100644 packages/chord/test/next/jobs/persisted.test.ts create mode 100644 packages/chord/test/next/store/allocator.test.ts create mode 100644 packages/chord/test/next/store/apply-batch.test.ts create mode 100644 packages/chord/test/next/store/claims.test.ts create mode 100644 packages/chord/test/next/store/refusal.test.ts create mode 100644 packages/durable/src/storage/store/commit.ts create mode 100644 packages/durable/src/storage/store/documents.ts create mode 100644 packages/durable/src/storage/store/index.ts create mode 100644 packages/durable/src/storage/store/queries.ts create mode 100644 packages/durable/src/storage/store/storage.ts create mode 100644 packages/durable/src/storage/store/tables.ts create mode 100644 packages/durable/test/examples/35-durable-on-the-store.ts create mode 100644 packages/durable/test/examples/36-durable-views.ts create mode 100644 packages/durable/test/store-storage-node.ts create mode 100644 packages/durable/test/store-storage-support.ts create mode 100644 packages/durable/test/store-storage.test.ts create mode 100644 packages/durable/vitest.store.config.ts diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index 5724e1a42..f475c4c8a 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -13,6 +13,7 @@ ### Added +- Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 4add7d22b..8e63d8d36 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -490,3 +490,96 @@ Observations: - Late migration chunks cost what early ones do (paging forward by the last key keeps each chunk's read bounded); the spec's estimate of about 15 µs per migrated row holds with room. - A bulk write's step is the one place a single commit holds every owner's commits for milliseconds; the guide says so. + +## Slice 6b: Durable on the store, persisted jobs (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), SQLite files in `tmpdir()`. HEAD is `b2fcfee08`, extracted at a path of the same +length (`/home/badlogic/pi-6h`); runs alternate HEAD and the build. + +**Durable's storage benchmark** (`packages/durable/test/storage.bench.ts`, `TIMING_SCALE`), `StoreStorage` (outbox off) +against `SqliteStorage` in the same process, mean ms per operation over three runs (p75 where the mean is one outlier): + +| Case | SqliteStorage | StoreStorage | | +| --- | --- | --- | --- | +| exact entry lookup | 0.0030 | 0.0030 | 0% | +| entry page scan (100) | 0.0942 | 0.0940 | 0% | +| filtered task scan (50) | 0.0206 (p75 0.0200) | 0.0220 (p75 0.0207) | +7% mean, +3% p75 | +| exact document address among many | 0.0037 | 0.0037 | 0% | +| document replay tail (0) | 0.0089 (p75 0.0084) | 0.0092 (p75 0.0085) | +3% mean, +1% p75 | +| document replay tail (16 / 128 / 1024) | 0.0300 / 0.1726 / 1.224 | 0.0270 / 0.1522 / 1.135 | -10% / -12% / -7% | +| ancient / recent historical read | 0.1681 / 0.1611 | 0.1541 / 0.1513 | -8% / -6% | +| fork-depth history scan (100) | 0.1177 (p75 0.1156) | 0.1208 (p75 0.1159) | +3% mean, 0% p75 | +| fork-depth head lookup | 0.0361 | 0.0332 | -8% | +| commit one entry (20 samples) | 0.0439 (p75 0.0416) | 0.0601 (p75 0.0373) | +37% mean, -10% p75 | +| commit 100 entries | 0.743 | 0.620 | -17% | +| commit mixed entry/task/submission/document | 0.195 | 0.187 | -4% | +| reopen and first exact read | 0.294 | 0.443 | +51% | + +Warm micro-benchmarks behind the same cases (each read 20,000 times; commits 10,000 after 2,000 warm-up): page scan +91.2 vs 90.6 µs, task scan 19.9 vs 20.1 µs, fork-depth scan 114.1 vs 114.9 µs, head lookup 35.3 vs 32.9 µs, document +address 3.57 vs 3.68 µs, replay tail (0) 8.41 vs 8.44 µs; one entry per commit 29.6 vs 27.8 µs mean, 18.4 vs 17.1 µs +median. What it took, beyond the first build's compiled reads, point lookups without a bound `LIMIT`, version columns +selected only past version 1 and the counting read guard: + +- **Reads select only the record.** A scan selected an entry's `commit_seq` too (only `entry(id)` needs it): 7% of a + page scan. +- **No index for head markers on every entry.** `entries.by_head (conversation_id, has_head, id)` cost about 5.5 µs per + commit (a raw insert: 39.7 vs 45.2 µs); `SqliteStorage` has a partial index (`WHERE head IS NOT NULL`), which the + table vocabulary does not have. The head markers are an `entry_heads` table instead, written only with an entry that + has one: a commit without one writes no second index, and a head lookup reads the newest head's id, then the entry. +- **A document record is one JSON text.** `meta` keeps the whole Durable record, its creation seq included; a read + selects `meta` (and `retired_at` only at a point). +- **A store already at the newest schema opens with one read** (schema version, encoding and metadata row), without + the migrations' write transaction: 84 → 71 µs on a fresh connection. + +Commit one entry: the mean is one sample (Maglev compiling the commit path on the first fixtures: 130 and 386 µs at +fixtures 25 and 30 of the first store round of a 60-fixture replay, none in the second round, no GC or deopt traced); +the median and p75 are lower than `SqliteStorage`'s. Reopen: a fresh connection on a copied file, measured in parts +(`openNodeSqliteDatabase` is the same for both): `SqliteStorage.open` 42-60 µs; `openStore` 63-71 µs (the schema read, +the catalog with its JSON parse) plus `StoreStorage.open` 75 µs (declaring six tables: their SQL rendered, claims; the +catalog check of the table preparation; `max(record_ids.id)` for the allocator, as §2.1 asks; compiling the point +reads), first read +3 µs: about 0.1-0.15 ms once per open, cold code, not on any hot path. + +**Statements per commit** (cost guard in `packages/durable/test/store-storage.test.ts`): an entry append and a document +delta take 4 each (`SqliteStorage`: 5 and about 7). + +**Memory** (`packages/durable/test/storage-memory.ts`): heap after seeding 0.16 → 0.30 MiB (1k), 0.17 → 0.32 MiB (10k): +a fixed cost of the store (catalog, compiled statements, claims, the store's code), the same at both scales; heap after +reads 0.13 → 0.16-0.18 MiB. Disk 0.61 → 0.58 MiB (1k), 4.34 → 4.38 MiB (10k). + +**Chord's commit gate** (`test/next/bench/store.ts`), HEAD → build, two alternating runs each, medians: + +| Case | HEAD | build | +| --- | --- | --- | +| (a) one document delta per commit | 13.6-13.7 µs | 13.8-14.0 µs (+2%) | +| (b) one table put per commit | 12.8-13.3 µs | 13.0-13.2 µs | +| (c) whole transactions (no watch / watched) | 17.2-17.3 / 17.1-17.2 µs | 17.4-17.7 / 17.2-17.7 µs (+2%) | +| (d) one in-memory document (median / burst) | 2.1 / 2.9-3.0 µs | 2.1 / 4.9 µs | +| heap after GC | 2940 KiB | 2996 KiB | + +The revisions index is a partial index on bases (`(document_id, seq) WHERE kind = 'base'`): a delta writes no index +entry (the first build's `(document_id, kind, seq)` cost (a) 15%). (d)'s burst is not the in-memory path: run alone +(only section (d), three alternating runs) HEAD gives 4.9-5.1 µs and the build 4.9-5.0 µs per commit, and 40 bursts of +1,000 in a fresh process 2.17 vs 2.18-2.38 µs; in the full gate, HEAD's in-memory path runs warm from the persisted +sections before it, the build's does not share as much of their code (`StoredChange`'s variants). + +**Durable's conversation view benchmark** (`test/conversation-view-bench.ts`, `PI_DURABLE_BENCH_STORAGE`), two runs +each, `SqliteStorage` → `StoreStorage`: append one entry per commit 56.1-57.2 → 56.2-57.0 µs with one watch, 61.0-62.1 +→ 62.7-63.1 µs with four (+2%); bursts of 50 884-888 → 891 µs; `pi.live` only 62.7-63.0 → 65.1-66.6 µs (+4-6%); a +long-lived watch over 20,000 appends 0.83-0.85 → 0.84-0.85 s CPU; heaps 21.5 → 21.7 MiB. + +**Fix round 1** (2026-10-28, after the build reviews): `StoreStorage`'s reads take one promise besides the store's guard +(no `#call` wrapper; `close` waits for its commits, a read changes nothing), their request errors in a tagged result. +Warm micro-benchmark (20,000 runs, two runs each, `SqliteStorage` → `StoreStorage`): filtered task scan 19.8 → 19.8-20.1 +µs, `entry(conversation, id)` over the fork chain 23.1 → 21.9-22.3 µs. `storage.bench.ts` (two runs): task scan mean +0.0207 → 0.0217-0.0218 ms (p75 0.0200-0.0203 → 0.0204-0.0206), fork-depth scan 0.1177-0.1188 → 0.1210-0.1242 (p75 +0.1154-0.1167 → 0.1160-0.1193), the rest as above. +`conversation-view-bench` "resume 20 back", three runs: CPU 4.23-4.47 → 4.50-5.14 ms for a 3 ms scenario (one sample +each; the head lookup and scans under it are at parity or faster in the micro-benchmark). The persisted-job fixes +(one invocation per call of `run`, held state folded in a `PendingOps`): start 55.5 → 55.2 µs, immediate state change +21.3 → 21.2 µs, deferred 3.2 → 3.5 µs (the fold), in memory unchanged. + +**Persisted jobs** (`test/next/bench/jobs.ts`, 7 bursts of 500, medians): a start (resolved after its commit) 8.4 µs in +memory, 55.5 µs persisted (two commits: the record and state created, then the reservation); a state change inside +`run` 2.7 µs in memory, 21.3 µs persisted (one delta commit, as the gate's (c)), 3.2 µs deferred (stored with the +outcome). diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index c2498ecba..87d395336 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -27,7 +27,7 @@ slice spec's section citations are for orientation, not a reading limit (owner, - **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, errors. `index.ts` for apps, `internal.ts` for Chord's own code. -- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a** (committed; 5 `4a5b45c45`, 6a 2026-10-27): `src/next/documents`, `state`, `host`, `jobs`, `link`, +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28): `src/next/documents`, `state`, `host`, `jobs`, `link`, `remote`, `store`; examples 01-25; benchmarks in `benchmarks.md`. - **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), @@ -161,19 +161,6 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. -### Slice 6b: Durable and jobs on the store (from slice 5 and L3a-d) - -Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and claims"); each entry below is taken there. - -- **Persisted jobs** (from slice 5): `persist: { resume }`, records and state documents in the owner's store (D5), the - restart limit (D6), `abort` stored (D7), deferred state (D8), the seal as a clean stop (R3), the `finish` failure - count persisted, the job fingerprint and the start trace carrier stored. -- **`StoreStorage`** (D1-D4, R1, R2, claims): Durable's `Storage` on the store, passing its conformance suite; one id - allocator (D3); Durable's checks in the step; fork copies resolved before the step (S11); `adopt` in the host's batch. -- **`self.follow`** (D9, R4): a Chord source over Durable's synchronous observers. Reaching Durable from other objects - (the harness in a registry entry) moves to slice 8. -- **The applier for batch apply**: every `StoreWrite` kind, `applyBatch`, one code path with import. - ### Slice 7: reload and rollout - **A native restart drains first** (owner, 2026-10-26): park new calls, await running calls (bounded by @@ -185,6 +172,9 @@ Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and c per routine invocation; holds unsubscribed at seal and re-bound by member name; keyed members' versions mapped per key; in-memory `migrate` wired to reload (6a builds the slot version check and the migrate); `rebuild` (key and column-type changes) if a case needs it. +- **From slice 6b:** a persisted run of another job version or fingerprint stays `blocked { version }` with its record + as stored (resolve it on reload); blocked persisted runs are retried on reload; runs of jobs the code no longer + declares (below). - **A run whose job the new code no longer declares** (slice 6a builder, 2026-10-27): 6a keeps it in its phase, waiting, not invoked (unreachable without reload); slice 7 decides `blocked` with a reason (design §15's `blocked { version }` family). @@ -233,6 +223,7 @@ Specified in `slice-6b.md` (decided 2026-10-27, D1-D9, R1-R4, "owner names and c ### Slice 9: telemetry - Spans for routine calls and job runs (from slice 5). +- The use of a persisted run's stored start trace carrier (`trace`, stored since slice 6b). - Round-3 cut to decide: C11 (`ctx.trace`). diff --git a/packages/chord/package.json b/packages/chord/package.json index 57dbb5406..895c16523 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -36,6 +36,11 @@ "types": "./dist/telemetry-sqlite/index.d.ts", "import": "./dist/telemetry-sqlite/index.js" }, + "./next": { + "source": "./src/next/index.ts", + "types": "./dist/next/index.d.ts", + "import": "./dist/next/index.js" + }, "./sqlite": { "source": "./src/sqlite/index.ts", "types": "./dist/sqlite/index.d.ts", diff --git a/packages/chord/src/next/documents/definition.ts b/packages/chord/src/next/documents/definition.ts index 77906479b..e9a2eecf1 100644 --- a/packages/chord/src/next/documents/definition.ts +++ b/packages/chord/src/next/documents/definition.ts @@ -12,6 +12,15 @@ import { validatorFor } from "../data/internal.ts"; /** A document root: a JSON object. */ export type JsonObject = { [key: string]: JsonValue }; +/** + * When a commit reaches the store (Durable's `CommitPersistence`): `immediate`, the default, before the commit + * resolves; `deferred` (a persisted run's state change, D8): adopted and published at once, stored with the next + * immediate change, the run's outcome, the object's close, or about `withinMs` (default 1,000 ms) later. + */ +export type CommitPersistence = + | { readonly kind: "immediate" } + | { readonly kind: "deferred"; readonly withinMs?: number }; + /** What a persisted document's `checkpointWhen` learns of its stored revisions. */ export type CheckpointInfo = { /** The deltas stored after its newest base, without the change being decided. */ diff --git a/packages/chord/src/next/documents/documents.ts b/packages/chord/src/next/documents/documents.ts index 6ca0f5e82..cf9a22596 100644 --- a/packages/chord/src/next/documents/documents.ts +++ b/packages/chord/src/next/documents/documents.ts @@ -6,7 +6,7 @@ import type { Context } from "../../context/index.ts"; import { ChordError } from "../data/index.ts"; import type { Batch } from "../state/internal.ts"; import type { ChordStore } from "../store/store.ts"; -import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "./definition.ts"; import { SealToken } from "./fence.ts"; import { type HandleOf, type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; import { KeyedSlot } from "./keyed-slot.ts"; @@ -23,7 +23,7 @@ export type DocumentsOwner = { readonly batch: Batch; /** The object's context: shared cold loads run under it (rule 11). */ readonly context: Context; - /** The host's store and the object's owner string in it (`:`, W7), when it has one. */ + /** The host's store and the object's owner name in it (its type, R2), when it has one. */ readonly store: { readonly store: ChordStore; readonly owner: string } | undefined; /** A document the code cannot read fails the live object `broken` (slice 6a §1.4); ignored when it is not live. */ fail(error: ChordError): void; @@ -137,13 +137,19 @@ export class Documents { /** * Runs `fn` as one transaction on the line under `token` and commits what it drafted (§9): a throw, an abort of - * `context`, the token's seal or the object's end discards everything. + * `context`, the token's seal or the object's end discards everything. `persistence` defers storing a run state's + * change (D8). */ - async tx(fn: (tx: Transaction) => R | Promise, token: SealToken, context: Context): Promise { + async tx( + fn: (tx: Transaction) => R | Promise, + token: SealToken, + context: Context, + persistence?: CommitPersistence, + ): Promise { this.batch.assertOutside("tx"); this.assertAdmitted(token); const release = await this.line.acquire(); - const tx = new Transaction(this, token, release, context); + const tx = new Transaction(this, token, release, context, persistence); try { this.assertAdmitted(token); this.#holder = tx; @@ -232,6 +238,6 @@ export class Documents { /** The store side of persisted documents named `name`; the attempt checked the store first (P3). */ #persistence(name: string): Persistence { const { store, owner } = this.#owner.store!; - return new Persistence(store, owner, name); + return new Persistence(store, owner, "chord", name); } } diff --git a/packages/chord/src/next/documents/index.ts b/packages/chord/src/next/documents/index.ts index e7568d8f0..eaae754eb 100644 --- a/packages/chord/src/next/documents/index.ts +++ b/packages/chord/src/next/documents/index.ts @@ -1,5 +1,5 @@ // Documents for apps: definitions, handles and transactions. -export { defineState, type JsonObject, type StateDefinition } from "./definition.ts"; +export { type CommitPersistence, defineState, type JsonObject, type StateDefinition } from "./definition.ts"; export type { HandleOf, KeyedHandle, SingletonHandle } from "./handles.ts"; export type { Tx } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/internal.ts b/packages/chord/src/next/documents/internal.ts index b2eee6b2b..1bd15a893 100644 --- a/packages/chord/src/next/documents/internal.ts +++ b/packages/chord/src/next/documents/internal.ts @@ -5,5 +5,13 @@ export { Documents, type DocumentsOwner } from "./documents.ts"; export { SealToken } from "./fence.ts"; export { type KeyDeclarer, Keyed, Singleton } from "./handles.ts"; export { type KeyedSlot, KeySource } from "./keyed-slot.ts"; -export type { AnyDefinition, DocumentSlot } from "./slot.ts"; +export { + type DocumentChange, + type Held, + Persistence, + type Stored, + type StoredChange, + storedAfter, +} from "./persistence.ts"; +export { type AnyDefinition, type DocumentSlot, type PersistedSlot, SlotDefinition } from "./slot.ts"; export type { Staged, Transaction } from "./transaction.ts"; diff --git a/packages/chord/src/next/documents/keyed-slot.ts b/packages/chord/src/next/documents/keyed-slot.ts index 6575fd8ec..6faec91f0 100644 --- a/packages/chord/src/next/documents/keyed-slot.ts +++ b/packages/chord/src/next/documents/keyed-slot.ts @@ -8,12 +8,20 @@ import { type Prepared, type Tracker, track } from "../../delta/index.ts"; import type { Watch } from "../../watch.ts"; import { ChordError } from "../data/index.ts"; import { asChordError } from "../data/internal.ts"; +import type { StagedWrite } from "../store/step.ts"; import { compareText, storedText } from "../store/text.ts"; import { AddressSource } from "./address-source.ts"; import type { JsonObject } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; -import type { Loaded, Persistence, Stored } from "./persistence.ts"; +import { + type DocumentChange, + type Loaded, + type Persistence, + type Stored, + type StoredChange, + storedAfter, +} from "./persistence.ts"; import { Resident } from "./residency.ts"; import { type AnyDefinition, type DocumentSlot, type PersistedSlot, SlotDefinition } from "./slot.ts"; @@ -222,6 +230,10 @@ export class KeyedSlot implements DocumentSlot, PersistedSlot { return this.#residents.get(key)?.stored; } + stage(key: string, change: DocumentChange, was: Stored | undefined, writes: StagedWrite[]): StoredChange { + return this.persistence.stage(key, this.definition.current, was, change, writes); + } + /** * Takes the attempt's declaration: a newer version migrates each document in memory on its next access (persisted * ones store a base with their next change); the pinned keys are watched, so they migrate now, all or none, and @@ -251,7 +263,13 @@ export class KeyedSlot implements DocumentSlot, PersistedSlot { return this.definition.current.initial(context); } - adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + change: StoredChange | undefined, + ): void { + const stored = change === undefined ? undefined : storedAfter(change); const source = this.#sources.get(key); const before = source?.peek(); tracker.adopt(prepared); diff --git a/packages/chord/src/next/documents/persistence.ts b/packages/chord/src/next/documents/persistence.ts index 7b6082286..3efc45308 100644 --- a/packages/chord/src/next/documents/persistence.ts +++ b/packages/chord/src/next/documents/persistence.ts @@ -3,7 +3,7 @@ // checked `migrate`, and the writes one commit stages for a changed or retired document. import type { Context } from "../../context/index.ts"; -import type { Prepared } from "../../delta/index.ts"; +import type { Op, Prepared } from "../../delta/index.ts"; import { changeDocument, createDocument, @@ -32,25 +32,35 @@ export type Stored = { /** A loaded document: its value at the definition's version, and what is stored of it. */ export type Loaded = { readonly value: JsonObject; readonly stored: Stored }; +/** One transaction's change of one document: its prepared draft (if changed), and whether it retires it. */ +export type DocumentChange = { readonly prepared: Prepared | undefined; readonly retires: boolean }; + +/** A run state's held ops (D8), stored before the next change's own; `base` when a held change asked for a base. */ +export type Held = { readonly ops: readonly Op[]; readonly base: boolean }; + /** * What one commit stores of a document, in the store writes' words: a retirement (nothing stored after it), a - * creation whose id the step assigns (`create`, its position among the commit's creations), or a change of the record - * `stored` was prepared on. + * creation whose id the step assigns (`create`: `created.id` once it applied), a change of the record `stored` was + * prepared on, or nothing yet (`held`: a deferred change of a run's state, D8, kept in memory over `stored`). */ export type StoredChange = | { readonly kind: "retire" } - | { readonly kind: "create"; readonly version: number; readonly index: number } - | { readonly kind: "change"; readonly stored: Stored }; + | { readonly kind: "create"; readonly version: number; readonly created: { id: number } } + | { readonly kind: "change"; readonly stored: Stored } + | { readonly kind: "held"; readonly stored: Stored; readonly withinMs: number }; -/** The documents of one persisted definition name on one object: `(chord, :, name, key)`. */ +/** + * The documents of one persisted name on one object: `(, namespace, name, key)`, the namespace `chord` for + * `self.state`, `chord.jobs` and `chord.job-states` for persisted job runs (slice 6b D5). + */ export class Persistence { readonly store: ChordStore; readonly #address: Omit; - /** `owner` is `:` (W7); `kind` the definition's name. */ - constructor(store: ChordStore, owner: string, kind: string) { + /** `owner` is the object's type (R2); `kind` the definition's or the job's name. */ + constructor(store: ChordStore, owner: string, namespace: string, kind: string) { this.store = store; - this.#address = { namespace: "chord", owner, kind }; + this.#address = { owner, namespace, kind }; } /** @@ -58,7 +68,7 @@ export class Persistence { * as the store's). Counted (`chord.store.loads`). */ async read(key: string, context: Context): Promise { - const address = this.#at(key); + const address = this.at(key); const document = await this.store.read(async (executor) => { const record = await findDocument(executor, address); return record === undefined ? undefined : loadDocument(executor, record); @@ -92,7 +102,7 @@ export class Persistence { /** What is stored of the live document at `key` without loading it (a retirement's find), `undefined` when absent. */ async find(key: string, context: Context): Promise { - const record = await this.store.read((executor) => findDocument(executor, this.#at(key)), context); + const record = await this.store.read((executor) => findDocument(executor, this.at(key)), context); if (record === undefined) return undefined; // The change counter is all a retirement needs; `deltasSinceBase` is not read for one. return { id: record.id, revision: record.revision, version: record.version, deltasSinceBase: 0 }; @@ -106,16 +116,16 @@ export class Persistence { /** * Stages one commit's writes of the document at `key` into `writes` (rule 8: serialized here, before the queue): * its retirement when `retired` is stored, then its creation, or the change of `stored` (a base when created, when - * the stored version is older, or when `checkpointWhen` asks; a delta otherwise). `creations` counts the commit's - * creations so far (the step's ids follow their order). Returns what the commit stores. + * the stored version is older, or when `checkpointWhen` asks; a delta otherwise), `held` ops stored before the + * change's own (D8: a held base makes it a base). Returns what the commit stores. */ stage( key: string, definition: AnyDefinition, stored: Stored | undefined, - change: { readonly prepared: Prepared | undefined; readonly retires: boolean }, + change: DocumentChange, writes: StagedWrite[], - creations: number, + held?: Held, ): StoredChange { const { prepared, retires } = change; if (retires && stored !== undefined) writes.push(retireDocument(stored.id, stored.revision)); @@ -123,34 +133,43 @@ export class Persistence { const { version, fingerprint } = definition; if (retires || stored === undefined) { const base = { version, fingerprint, value: prepared.value }; - writes.push(createDocument(this.#at(key), "latest", base, JSON.stringify(prepared.value))); - return { kind: "create", version, index: creations }; + const created = { id: 0 }; + const write = createDocument(this.at(key), base, JSON.stringify(prepared.value), (id) => { + created.id = id; + }); + writes.push(write); + return { kind: "create", version, created }; } const info = { deltasSinceBase: stored.deltasSinceBase }; - const base = stored.version !== version || definition.checkpointWhen(prepared.value, prepared.ops, info); + const base = + held?.base === true || + stored.version !== version || + definition.checkpointWhen(prepared.value, prepared.ops, info); + const ops = held === undefined ? prepared.ops : [...held.ops, ...prepared.ops]; const content: DocumentContent = base ? { kind: "base", version, fingerprint, value: prepared.value } - : { kind: "delta", version, ops: prepared.ops }; - const text = JSON.stringify(base ? prepared.value : prepared.ops); + : { kind: "delta", version, ops }; + const text = JSON.stringify(base ? prepared.value : ops); writes.push(changeDocument(stored.id, stored.revision, content, text)); const deltasSinceBase = base ? 0 : stored.deltasSinceBase + 1; return { kind: "change", stored: { id: stored.id, revision: stored.revision + 1, version, deltasSinceBase } }; } /** The address of `key`: a singleton's (`""`) has no key. */ - #at(key: string): DocumentAddress { + at(key: string): DocumentAddress { return key === "" ? this.#address : { ...this.#address, key }; } } -/** What is stored after a commit's `change` was adopted with the commit's created ids. */ -export function storedAfter(change: StoredChange, ids: readonly number[]): Stored | undefined { +/** What is stored after a commit's `change` was adopted. */ +export function storedAfter(change: StoredChange): Stored | undefined { switch (change.kind) { case "retire": return undefined; case "create": - return { id: ids[change.index]!, revision: 1, version: change.version, deltasSinceBase: 0 }; + return { id: change.created.id, revision: 1, version: change.version, deltasSinceBase: 0 }; case "change": + case "held": return change.stored; } } diff --git a/packages/chord/src/next/documents/slot.ts b/packages/chord/src/next/documents/slot.ts index 657e8d90d..43d0e01cd 100644 --- a/packages/chord/src/next/documents/slot.ts +++ b/packages/chord/src/next/documents/slot.ts @@ -6,11 +6,12 @@ import type { Context } from "../../context/index.ts"; import { type Prepared, type Tracker, track } from "../../delta/index.ts"; import { ChordError } from "../data/index.ts"; +import type { StagedWrite } from "../store/step.ts"; import { AddressSource } from "./address-source.ts"; -import type { JsonObject, StateDefinition } from "./definition.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; -import type { Persistence, Stored } from "./persistence.ts"; +import { type DocumentChange, type Persistence, type Stored, type StoredChange, storedAfter } from "./persistence.ts"; /** A document definition, whatever its value and keyedness. */ export type AnyDefinition = StateDefinition; @@ -27,8 +28,13 @@ export interface DocumentSlot { initial(context: Context): JsonObject; /** The persisted slot's store side; `undefined` for documents in memory only. */ readonly persisted: PersistedSlot | undefined; - /** Adopts a commit of the document at `key` (inside the batch), with what is stored of it after the commit. */ - adopt(key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void; + /** Adopts a commit of the document at `key` (inside the batch), with what the commit stored of it. */ + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + stored: StoredChange | undefined, + ): void; /** Retires the document at `key` (inside the batch). */ retire(key: string): void; } @@ -41,6 +47,17 @@ export interface PersistedSlot { cold(key: string): boolean; /** What is stored of the document at `key` as memory knows it; `undefined` when absent or cold. */ stored(key: string): Stored | undefined; + /** + * Stages one commit's writes of the document at `key` into `writes` (`was`: what is stored of it, a cold one's as + * the transaction found it); `persistence` is the transaction's (a deferred change of a run's state is held, D8). + */ + stage( + key: string, + change: DocumentChange, + was: Stored | undefined, + writes: StagedWrite[], + persistence: CommitPersistence, + ): StoredChange; /** * Loads the document at `key` (its caller holds the line): installed when present and still absent from memory, * and, for a transaction's load (`token`), while the token is not sealed (a cut transaction released the line, so @@ -205,6 +222,10 @@ export class SingletonSlot extends AddressSource implements DocumentSlot, Persis return this.#stored; } + stage(key: string, change: DocumentChange, was: Stored | undefined, writes: StagedWrite[]): StoredChange { + return this.persistence.stage(key, this.definition.current, was, change, writes); + } + current(_key: string): Tracker | undefined { return this.#tracker; } @@ -214,12 +235,17 @@ export class SingletonSlot extends AddressSource implements DocumentSlot, Persis return this.#initial!; } - adopt(_key: string, tracker: Tracker, prepared: Prepared, stored: Stored | undefined): void { + adopt( + _key: string, + tracker: Tracker, + prepared: Prepared, + stored: StoredChange | undefined, + ): void { // Unchecked: a step admitted before the object ended adopts after its seal (slice 6a §1.3). const before = this.#tracker?.value ?? this.#initial!; tracker.adopt(prepared); this.#tracker = tracker; - this.#stored = stored; + this.#stored = stored === undefined ? undefined : storedAfter(stored); this.written(before, prepared); } diff --git a/packages/chord/src/next/documents/transaction.ts b/packages/chord/src/next/documents/transaction.ts index 0dfb6c1bb..506c15468 100644 --- a/packages/chord/src/next/documents/transaction.ts +++ b/packages/chord/src/next/documents/transaction.ts @@ -9,17 +9,18 @@ import type { Job } from "../jobs/internal.ts"; import type { Columns, PutRowOf, RowOf } from "../store/columns.ts"; import type { Condition, SetOf } from "../store/expression.ts"; import type { ColumnRefs, Query } from "../store/query.ts"; +import { refusalOf } from "../store/refusal.ts"; import type { StagedWrite } from "../store/step.ts"; import type { ChordStore } from "../store/store.ts"; import type { KeyOf } from "../store/table.ts"; import { TableChanges } from "../store/table-changes.ts"; import { type Page, type PageOptions, ROWS_PAGE, type Table, TableHandle } from "../store/table-handle.ts"; -import type { JsonObject } from "./definition.ts"; +import type { CommitPersistence, JsonObject } from "./definition.ts"; import type { Documents } from "./documents.ts"; import type { SealToken } from "./fence.ts"; import type { Keyed, KeyedHandle, Singleton, SingletonHandle } from "./handles.ts"; import type { KeyedSlot } from "./keyed-slot.ts"; -import { type Stored, type StoredChange, storedAfter } from "./persistence.ts"; +import type { Stored, StoredChange } from "./persistence.ts"; import type { DocumentSlot, SingletonSlot } from "./slot.ts"; /** What `self.tx` gives its callback. Inside it: only `await tx.*`, no other `self.tx`, no calls to other objects. */ @@ -98,6 +99,8 @@ type Entry = { found: Stored | undefined; }; +const IMMEDIATE: CommitPersistence = { kind: "immediate" }; + /** * One document change a commit adopts: a removal, a draft's prepared change, or both; and for a persisted document * what the commit stores of it. @@ -109,9 +112,6 @@ type Adoption = { readonly stored: StoredChange | undefined; }; -/** The created documents' ids of a commit without store writes: none. */ -const NO_IDS: readonly number[] = []; - /** * One transaction of an object, under the seal token of whoever runs it (an attempt, a job run's invocation, the * object). Commit prepares every draft (Delta records ops and checks only placed values) and adopts the changed @@ -135,15 +135,33 @@ export class Transaction implements Tx { #tables: Map | undefined; /** `tx.*` reads underway: a callback settling with one is refused (rule 3, as Durable's). */ #pending = 0; + /** When its commit reaches the store (a run state's deferred change, D8). */ + readonly #persistence: CommitPersistence; #state: "open" | "committing" | "stepping" | "committed" | "discarded" | "cut" = "open"; /** Rejected by `cut` while the commit waits in the store's queue: its caller is told at once (rule 3). */ #queued: PromiseWithResolvers | undefined; - constructor(documents: Documents, token: SealToken, release: () => void, context: Context) { + constructor( + documents: Documents, + token: SealToken, + release: () => void, + context: Context, + persistence: CommitPersistence = IMMEDIATE, + ) { this.#documents = documents; this.token = token; this.#release = release; this.#context = context; + this.#persistence = persistence; + } + + /** + * The store writes of this transaction, which `write`s join (Chord's own documents: a job's run records): made by the + * first, which sends the commit through `store`'s step. + */ + stageStore(store: ChordStore): StagedWrite[] { + this.#writes ??= { store, staged: [] }; + return this.#writes.staged; } /** The object whose transaction this is. */ @@ -311,7 +329,6 @@ export class Transaction implements Tx { this.#state = "committing"; this.#context.abortSignal?.throwIfAborted(); const adoptions: Adoption[] = []; - let creations = 0; for (const entries of this.#entries.values()) { for (const entry of entries.values()) { let prepared = entry.draft?.change.prepare(); @@ -325,19 +342,16 @@ export class Transaction implements Tx { const persisted = slot.persisted; let stored: StoredChange | undefined; if (persisted !== undefined) { - const { persistence, definition } = persisted; - this.#writes ??= { store: persistence.store, staged: [] }; - const change = { prepared, retires }; + const writes = this.stageStore(persisted.persistence.store); const was = persisted.stored(key) ?? entry.found; - stored = persistence.stage(key, definition.current, was, change, this.#writes.staged, creations); - if (stored.kind === "create") creations++; + stored = persisted.stage(key, { prepared, retires }, was, writes, this.#persistence); } adoptions.push({ entry, prepared, retires, stored }); } } const writes = this.#writes; if (writes === undefined) { - this.#adopt(adoptions, NO_IDS); + this.#adopt(adoptions); this.#state = "committed"; return; } @@ -356,7 +370,7 @@ export class Transaction implements Tx { this.#state = "stepping"; return true; }, - adopt: (_seq: number, ids: readonly number[]) => this.#adopt(adoptions, ids), + adopt: () => this.#adopt(adoptions), }; this.#queued = Promise.withResolvers(); try { @@ -364,7 +378,7 @@ export class Transaction implements Tx { await Promise.race([store.commit(staged, fence, withoutAbortSignal(this.#context)), this.#queued.promise]); } catch (error) { if (this.#state === "cut") throw this.#cutError(); - throw error; + throw this.#refused(error); } finally { this.#queued = undefined; } @@ -372,18 +386,29 @@ export class Transaction implements Tx { } /** - * Adopts the changed documents (persisted ones with what is stored of them, `ids` being the commit's created - * documents' ids) and the staged changes in one batch; nothing when nothing changed. + * The error a commit the store rejected rejects with (D4): the store's own (`released`, `storage_failed`) as it + * is; a refused step classified, an `int` `CHECK` failing only this transaction (`invalid/args`), any other refusal + * failing the live object `broken` too (memory and disk disagree). */ - #adopt(adoptions: readonly Adoption[], ids: readonly number[]): void { + #refused(error: unknown): unknown { + const refusal = refusalOf(error, this.#documents.label); + if (refusal === undefined) return error; + if (refusal.kind === "broken") this.#documents.fail(refusal.error); + return refusal.error; + } + + /** + * Adopts the changed documents (persisted ones with what the commit stored of them) and the staged changes in one + * batch; nothing when nothing changed. + */ + #adopt(adoptions: readonly Adoption[]): void { const staged = this.#staged; if (adoptions.length === 0 && staged === undefined) return; this.#documents.batch.commit(() => { for (const { entry, prepared, retires, stored } of adoptions) { if (retires) entry.slot.retire(entry.key); if (prepared === undefined) continue; - const after = stored === undefined ? undefined : storedAfter(stored, ids); - entry.slot.adopt(entry.key, entry.draft!.tracker, prepared, after); + entry.slot.adopt(entry.key, entry.draft!.tracker, prepared, stored); } for (const each of staged?.values() ?? []) each.adopt(); }, this.#context); diff --git a/packages/chord/src/next/host/attempt.ts b/packages/chord/src/next/host/attempt.ts index aa9071dbd..a424b5ff6 100644 --- a/packages/chord/src/next/host/attempt.ts +++ b/packages/chord/src/next/host/attempt.ts @@ -5,7 +5,7 @@ import { awaitWithContext, type Context, withAbortSignal } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; -import type { NativeWatchable, Watchable } from "../../watch.ts"; +import type { NativeWatchable, Watchable, WatchEnd } from "../../watch.ts"; import type { Contract, Implementation, ParamsOf } from "../contracts/index.ts"; import { contractData, type ShapeOf } from "../contracts/internal.ts"; import { ChordError, type FailureCause, type Principal } from "../data/index.ts"; @@ -30,6 +30,7 @@ import { type Table, TableHandle } from "../store/table-handle.ts"; import { prepareTables } from "../store/table-migration.ts"; import { CallContext, type Caller, type Ctx, withLinkedAbortSignal } from "./ctx.ts"; import { Deadline } from "./deadline.ts"; +import { type FollowAttach, FollowSource } from "./follow.ts"; import type { HostError } from "./host.ts"; import { attachDerives, checkImplementation, type KeyedBinding, type Members } from "./members.ts"; import { @@ -77,6 +78,13 @@ export interface Self

extends RoutineDeclaratio ): Native; /** Runs `fn` as one transaction of this object (§9). */ tx(fn: (tx: Tx) => R | Promise, context: Context): Promise; + /** + * A source following another runtime's synchronous observer (Durable's views, slice 6b §3): `attach` runs during + * activation once the natives opened (a rejection or `undefined` fails the activation `transient`), and the observer + * is detached when the attempt ends. Its runtime failing fails this object `transient`; its `ended` is final, the + * source keeping its last value. A state member, a derive source, an `on` source. + */ + follow(attach: FollowAttach): Watchable; /** * This object's job `definition` (§15), once per job name: `run` runs each started run outside transactions, * `finish` records its outcome in one. Starts are taken once the object is live. @@ -143,6 +151,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { #members: Members | undefined; #variant: string | undefined; readonly #derives: Derive[] = []; + /** Its followed sources, attached in declaration order after the natives. */ + readonly #follows: FollowSource[] = []; readonly #routines = new Routines(); readonly #jobs = new Map(); /** The tables it declared, by name. */ @@ -162,6 +172,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { #previous: Attempt | undefined; /** What activation returns once the attempt was abandoned: a timeout, a cycle through its object, or its end. */ #abandoned: Exclude | undefined; + /** A followed source's end while this attempt activated: it fails as it goes live (if its activation returned). */ + #endedEarly: ChordError | undefined; constructor(object: ChordObject, previous: Attempt | undefined, context: Context) { this.#object = object; @@ -173,8 +185,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { state>(definition: D): HandleOf { this.#assertDeclaring("state"); const what = `state "${definition.name}"`; - if (definition.persist) this.#assertPersistable(what); if (this.#declared.has(definition.name)) throw new Error(`${this.#object.label}: ${what} is declared twice`); + if (definition.persist) { + this.#assertPersistable(what); + this.#object.claimDocuments("chord"); + } this.#declared.set(definition.name, definition); return this.#object.documents.declare(definition, this.#token, this); } @@ -189,15 +204,16 @@ export class Attempt implements StoreSelf, KeyDeclarer { table(definition: TableDefinition): Table { this.#assertDeclaring("table"); const object = this.#object; - const { host, served, label } = object; + const { served, label } = object; const store = this.#assertPersistable(`table "${definition.name}"`); if (this.#tables.has(definition.name)) throw new Error(`${label}: table "${definition.name}" is declared twice`); - const sqlName = tableSqlName(host.address?.kind ?? "client", served.contract.name, definition.name); + const sqlName = tableSqlName(served.contract.name, definition.name); + object.claimTable(sqlName); const owner = { label, scope: object.documents, assertLive: () => object.documents.assertLive(), - fail: (error: ChordError) => object.failLive(this, error), + fail: (error: ChordError) => object.failLive(this, { kind: "broken", error }), }; // The handle's rows are the definition's, typed by `C` and `K`. const handle = new TableHandle(definition as unknown as TableDefinition, sqlName, store, owner); @@ -249,6 +265,16 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#object.documents.tx(fn, this.#token, context); } + follow(attach: FollowAttach): Watchable { + this.#assertDeclaring("follow"); + const object = this.#object; + const owner = { label: object.label, context: this.#context, ended: (end: WatchEnd) => this.#followEnded(end) }; + // The source holds what the runtime gives it, `T`, as Chord's JSON sources do. + const source = new FollowSource(attach as unknown as FollowAttach, owner, object.host.batch); + this.#follows.push(source); + return source as unknown as Watchable; + } + on(source: Watchable, fn: OnHandler): void; on(source: NativeWatchable, fn: OnNativeHandler): void; on(source: Watchable | NativeWatchable, fn: (...args: never[]) => void | Promise): void { @@ -267,6 +293,11 @@ export class Attempt implements StoreSelf, KeyDeclarer { if (this.#jobs.has(definition.name)) { throw new Error(`${object.label}: job "${definition.name}" is declared twice`); } + if (definition.persist !== undefined) { + this.#assertPersistable(`job "${definition.name}"`); + object.claimDocuments("chord.jobs"); + object.claimDocuments("chord.job-states"); + } const owner = { host: object.host, caller: this.#caller(), @@ -314,9 +345,34 @@ export class Attempt implements StoreSelf, KeyDeclarer { this.#abort.abort(outcome.kind === "ended" ? outcome.error : outcome.cause.error); } + /** + * A followed runtime ended its observer (R4): `failed` fails this attempt `transient`, live or activating; `ended` + * does only while it activates (it would go live over a source that ended). An activation that already returned + * fails as it goes live. + */ + #followEnded(end: WatchEnd): void { + if (end.kind === "ended" && this.#isLive) return; + const { label } = this.#object; + const error = + end.kind === "failed" + ? asChordError(end.error, label) + : new ChordError({ code: "unavailable", message: `${label}: a followed source ended while it activated` }); + if (this.#isLive) { + this.#object.failLive(this, { kind: "transient", error }); + return; + } + this.#endedEarly ??= error; + this.abandon({ kind: "failed", cause: { kind: "transient", error } }); + } + /** The object went live: its jobs take starts and its routines start, in this turn; both stop when it is sealed. */ live(): void { this.#isLive = true; + const early = this.#endedEarly; + if (early !== undefined) { + this.#object.failLive(this, { kind: "transient", error: early }); + return; + } for (const job of this.#jobs.values()) job.live(); const { host, label } = this.#object; this.#routines.start({ @@ -421,6 +477,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { if ((await drained(this.#running.keys(), this.#running.size, host.timeouts.close)) === "timeout") { host.report({ kind: "drain_timeout", object: label }); } + // The close writer: the persisted runs the seal stopped, and the held state changes (slice 6b §4.2). + for (const job of this.#jobs.values()) await job.writeAtClose(this.#context); // Work stuck past the drain keeps running, but no longer counts as a call or holds its `ctx.get` references. for (const [running, cleanup] of this.#running) { this.#running.delete(running); @@ -434,10 +492,14 @@ export class Attempt implements StoreSelf, KeyDeclarer { return failures; } - /** Detaches its derives (once their object no longer needs them): reads throw `released`, their watches end. */ + /** + * Detaches its derives (once their object no longer needs them: reads throw `released`, their watches end) and its + * followed sources (their observers detached, their watches ended). + */ detach(): void { const ended = this.#object.ended(); for (const derive of this.#derives) derive.detach(ended); + for (const follow of this.#follows) follow.detach(); } /** Releases its references, and the previous attempt's if this one never bound its own. */ @@ -458,6 +520,8 @@ export class Attempt implements StoreSelf, KeyDeclarer { const loading = documents.load(this.#opening); if (loading !== undefined) await loading; if (this.#tables.size > 0) await this.#prepareTables(deadline); + // A persisted job's runs, loaded once per object: rebuilt before anything runs. + for (const job of this.#jobs.values()) await job.load(this.#opening); const { host } = this.#object; dependencies = this.#references.map(({ binding, contract, params }) => host.bind(binding, contract, params, this.#opening), @@ -508,6 +572,15 @@ export class Attempt implements StoreSelf, KeyDeclarer { return this.#abandoned ?? { kind: "failed", cause: openFailure(error, this.#object.label) }; } } + for (const follow of this.#follows) { + if (this.#abandoned !== undefined) return this.#abandoned; + try { + await awaitWithContext(follow.attach(this.#opening), this.#opening); + } catch (error) { + const cause: FailureCause = { kind: "transient", error: asChordError(error, this.#object.label) }; + return this.#abandoned ?? { kind: "failed", cause }; + } + } try { const checked = checkImplementation(this.#object.served.contract, implementation); this.#members = checked.members; diff --git a/packages/chord/src/next/host/follow.ts b/packages/chord/src/next/host/follow.ts new file mode 100644 index 000000000..d4131ca3d --- /dev/null +++ b/packages/chord/src/next/host/follow.ts @@ -0,0 +1,248 @@ +// Followed sources (slice 6b D9, R4, §3): `self.follow(attach)`, a Chord source over a synchronous observer of another +// runtime (Durable's `observeConversation`, `observeTaskGraph`, `observeDoc`). Attached during its attempt's +// activation, detached at the attempt's end. Calls inside a running batch (Durable adopting in the store's commit step) +// are marked in that batch, calls outside one are grouped into one batch on the next microtask, so a derive over two +// views of one commit computes once with both changes. + +import type { Context } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Watch, WatchCall, WatchEnd } from "../../watch.ts"; +import { ChordError } from "../data/index.ts"; +import { type Batch, Source, type Subscriber } from "../state/internal.ts"; + +/** The observer a followed runtime is given: Chord's watch shapes (Durable's `Observer`). */ +export type FollowObserver = { + /** One change, synchronously and quickly: Chord only records it. */ + advance(call: WatchCall, context: Context): void; + /** The followed source ended: `failed` (its runtime failed), or `ended` for good. */ + end(end: WatchEnd): void; +}; + +/** What attaching gives: the value every later call starts from, and how to detach (Durable's `Observation`). */ +export type FollowAttachment = { readonly value: T; detach(): void }; + +/** + * What `self.follow` takes: attaches `observer`, resolving with the attachment, `undefined` for an absent address. + * The observer takes calls of any value (it only records them), so `T` is inferred from the attachment alone. + */ +export type FollowAttach = ( + observer: FollowObserver, + context: Context, +) => Promise | undefined>; + +/** Who a followed source belongs to: its attempt (R4) and its context. */ +export type FollowOwner = { + readonly label: string; + readonly context: Context; + /** + * The followed runtime ended its observer: `failed` fails the attempt `transient` (live or activating, its retry + * attaches again); `ended` while the attempt activates fails it `transient` too (it would go live on a source that + * ended); `ended` while live is final for the source only. + */ + ended(end: WatchEnd): void; +}; + +/** + * Where a followed source stands: waiting for its attachment (with the runtime's end when it came first), attached, + * ended by the runtime (its value stays; detached at its attempt's end), or detached by its attempt. + */ +type FollowState = + | { readonly kind: "attaching"; readonly end: WatchEnd | undefined } + | { readonly kind: "attached"; readonly detach: () => void } + | { readonly kind: "ended"; readonly detach: () => void } + | { readonly kind: "detached" }; + +const NO_OPS: readonly Op[] = []; + +const ENDED: WatchEnd = { kind: "ended" }; + +/** A followed source: the runtime's value, its calls applied in order, its frames delivered in the host's batch. */ +export class FollowSource extends Source { + readonly #attach: FollowAttach; + readonly #owner: FollowOwner; + #state: FollowState = { kind: "attaching", end: undefined }; + #value: JsonValue | undefined; + /** Calls not applied yet: before the attachment resolved, and outside a batch until the next microtask. */ + #queued: WatchCall[] = []; + /** Whether a microtask is queued to apply `#queued` in a batch of its own. */ + #scheduled = false; + /** + * The runtime's end, still to reach the watches: `queued` after the queued calls' batch, `frame` after the current + * batch's frame (a retirement and an end in one batch: the retirement first). + */ + #ending: "none" | "queued" | "frame" = "none"; + /** The ops of the current batch's frame (kept only while watched), and whether a retirement came in it. */ + #ops: Op[] = []; + #retired = false; + #listed = false; + + constructor(attach: FollowAttach, owner: FollowOwner, batch: Batch) { + super(batch); + this.#attach = attach; + this.#owner = owner; + } + + get value(): JsonValue { + if (this.#value === undefined) { + throw new ChordError({ + code: "not_ready", + message: `${this.#owner.label}: a followed source read before it attached`, + }); + } + return this.#value; + } + + /** + * Attaches (activation, after the natives opened): the calls that came meanwhile apply on the attachment's value, + * and an end that came meanwhile takes effect then (the value stays). A rejected attachment, or `undefined` (an + * absent address), throws; an attachment arriving after its attempt detached the source is detached at once. + */ + async attach(context: Context): Promise { + const observer: FollowObserver = { + advance: (call) => this.#advance(call), + end: (end) => this.#end(end), + }; + const attached = await this.#attach(observer, context); + if (attached === undefined) { + throw new ChordError({ code: "unavailable", message: `${this.#owner.label}: the followed address is absent` }); + } + const state = this.#state; + const detach = () => attached.detach(); + if (state.kind !== "attaching") { + detach(); + return; + } + this.#value = attached.value; + // Nothing watches it yet (members are checked after this): the queued calls only move the value on. + for (const call of this.#queued) if (call.kind === "value") this.#value = call.value; + this.#queued = []; + this.#state = state.end?.kind === "ended" ? { kind: "ended", detach } : { kind: "attached", detach }; + } + + /** Its attempt ended: the observer is detached (also after the runtime ended it) and its watches end `ended`. */ + detach(): void { + const state = this.#state; + this.#state = { kind: "detached" }; + this.#queued = []; + if (state.kind === "attached" || state.kind === "ended") state.detach(); + this.endWatches(); + } + + /** A watch of an ended source opens at its value and ends `ended` at once. */ + override openWatch(context: Context): Watch { + const watch = super.openWatch(context); + if (this.#over) watch.end(ENDED); + return watch; + } + + /** A subscriber of an ended source (a reference's watch through its member source) ends `ended` at once. */ + override subscribe(subscriber: Subscriber): void { + if (this.#over) subscriber.end(ENDED); + else super.subscribe(subscriber); + } + + deliver(context: Context): void { + const ops = this.#ops; + const retired = this.#retired; + this.#ops = []; + this.#retired = false; + this.#listed = false; + this.publish(this.#value!, ops.length === 0 ? NO_OPS : ops, retired, context); + if (this.#ending === "frame") { + this.#ending = "none"; + this.endWatchesAfterDelivery(); + } + } + + /** Whether nothing reaches its watches any more: ended by the runtime (what it gave delivered) or detached. */ + get #over(): boolean { + const { kind } = this.#state; + return (kind === "ended" && this.#ending === "none") || kind === "detached"; + } + + /** + * A call of the runtime: before the attachment, queued; inside a running batch, applied there (after what was + * queued); outside one, queued for one batch on the next microtask. + */ + #advance(call: WatchCall): void { + const { kind } = this.#state; + if (kind === "ended" || kind === "detached") return; + if (kind === "attaching" || !this.batch.running) { + this.#queued.push(call); + if (kind === "attached") this.#schedule(); + return; + } + this.#applyQueued(); + this.#apply(call); + } + + /** + * The runtime ended its observer (R4); its attempt is told. `ended` is final: what came before reaches the watches + * first (the queued calls in their batch, a call of the running batch in its frame), then they end, the value + * staying. A `failed` attempt detaches the source. + */ + #end(end: WatchEnd): void { + const state = this.#state; + if (state.kind === "ended" || state.kind === "detached") return; + if (state.kind === "attaching") { + if (state.end === undefined) this.#state = { kind: "attaching", end }; + this.#owner.ended(end); + return; + } + if (end.kind === "ended") { + this.#state = { kind: "ended", detach: state.detach }; + if (this.#queued.length > 0 && !this.batch.running) this.#ending = "queued"; + else { + // Inside a running batch the queued calls join it (none is queued outside one: they wait for their batch). + this.#applyQueued(); + this.#endAfterFrame(); + } + } + this.#owner.ended(end); + } + + /** The watches end once this batch's frame reached them, at once when the source is in none. */ + #endAfterFrame(): void { + // No frame to wait for (nothing watches it): the end is over now, and a later watch ends at once. + this.#ending = this.#listed ? "frame" : "none"; + if (!this.#listed) this.endWatchesAfterDelivery(); + } + + #schedule(): void { + if (this.#scheduled) return; + this.#scheduled = true; + this.batch.soon(() => { + this.#scheduled = false; + const { kind } = this.#state; + if (kind === "attached" || kind === "ended") this.#applyQueued(); + if (this.#ending === "queued") this.#endAfterFrame(); + }, this.#owner.context); + } + + #applyQueued(): void { + const queued = this.#queued; + if (queued.length === 0) return; + this.#queued = []; + for (const call of queued) this.#apply(call); + } + + /** Applies one value call inside the running batch: the new value, its ops for the frame, the source marked once. */ + #apply(call: WatchCall): void { + // The runtime's failures are final: it ends its observer instead of sending one. + if (call.kind !== "value") return; + this.#value = call.value; + if (call.retired) this.#retired = true; + if (!this.watched) { + this.changed(); + if (call.retired) this.retiredInBatch(); + return; + } + for (const op of call.ops) this.#ops.push(op); + if (!this.#listed) { + this.#listed = true; + this.changed(); + } + if (call.retired) this.retiredInBatch(); + } +} diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index 8312e70bd..ecfad05ed 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -126,6 +126,14 @@ export type HostError = readonly id: string; readonly error: unknown; } + /** A persisted run was cut off five times in a row by crashes (D6): it ends `failed { restart_limit }`, `finish` runs. */ + | { + readonly kind: "job_interrupted"; + readonly object: string; + readonly job: string; + readonly id: string; + readonly restarts: number; + } /** A blocked run was dropped without `finish`: aborted, or its object ended. */ | { readonly kind: "job_dropped"; readonly object: string; readonly job: string; readonly id: string }; @@ -303,7 +311,7 @@ export class ChordHost

this.#storageFailed(error)); + this.store?.attach({ failed: (error) => this.#storageFailed(error), batch: this.batch }); } async load(facets: readonly { readonly name: string; readonly facet: Facet

}[], context: Context): Promise { diff --git a/packages/chord/src/next/host/index.ts b/packages/chord/src/next/host/index.ts index 6eb1a040c..3d7611c5d 100644 --- a/packages/chord/src/next/host/index.ts +++ b/packages/chord/src/next/host/index.ts @@ -5,6 +5,7 @@ export type { ActivityCounts } from "./activity.ts"; export type { Factory, Self, StoreSelf } from "./attempt.ts"; export type { Ctx } from "./ctx.ts"; export { defineFacet, type Facet, type FacetHost, type ServeOptions } from "./facet.ts"; +export type { FollowAttach, FollowAttachment, FollowObserver } from "./follow.ts"; export { createHost, type GetOptions, diff --git a/packages/chord/src/next/host/object.ts b/packages/chord/src/next/host/object.ts index ebdf37b4a..43e4d80fd 100644 --- a/packages/chord/src/next/host/object.ts +++ b/packages/chord/src/next/host/object.ts @@ -142,6 +142,8 @@ export class ChordObject implements Target { readonly #members = new Map(); /** The keys of its keyed members that references hold, across attempts. */ readonly #keys: KeyedMembers; + /** The one timer storing its jobs' held state changes by their earliest deadline (D8), while one is held. */ + #flush: { readonly deadline: number; readonly timer: ReturnType } | undefined; constructor( host: AnyHost, @@ -167,12 +169,9 @@ export class ChordObject implements Target { label: this.label, batch: host.batch, context, - store: - store === undefined - ? undefined - : { store, owner: `${host.address?.kind ?? "client"}:${served.contract.name}` }, + store: store === undefined ? undefined : { store, owner: served.contract.name }, fail: (error) => { - if (this.#state.kind === "live") this.failLive(this.#state.attempt, error); + if (this.#state.kind === "live") this.failLive(this.#state.attempt, { kind: "broken", error }); }, }); } @@ -327,12 +326,15 @@ export class ChordObject implements Target { jobRecords(definition: JobDefinition): JobRecords { const records = this.#jobs.get(definition.name); if (records === undefined) { + const { store } = this.host; const owner = { label: this.label, host: this.host, context: this.#context, documents: this.documents, + store: store === undefined ? undefined : { store, owner: this.served.contract.name }, countRuns: (delta: number) => this.countRuns(delta), + held: (deadline: number) => this.#heldUntil(deadline), }; const made = new JobRecords(definition, owner); this.#jobs.set(definition.name, made); @@ -347,19 +349,34 @@ export class ChordObject implements Target { } /** - * A runtime failure of the live attempt `attempt` (a `migrate` failing on access, a stored version newer than the - * code): the attempt is sealed and the object `failed` with the broken `cause`; calls reject with it, holders are - * told, the state members keep their values once the admitted commits adopted, the natives close in the background - * (then its references are released), `onError` gets `object_failed` once. Documents and job runs stay. Recovery: - * `ref.retry()` (a new attempt, whose members the member sources follow once it is live). A failure of a sealed - * attempt is ignored. + * Claims table `table` (its SQL name) in the host's store for this object, until its end (slice 6b claims); held by + * another owner: `invalid/args`, which fails the declaring factory. */ - failLive(attempt: Attempt, error: ChordError): void { + claimTable(table: string): void { + this.host.store!.claims.table(table, this); + } + + /** Claims this object's documents in `namespace` of the host's store, as `claimTable`. */ + claimDocuments(namespace: string): void { + this.host.store!.claims.documents(this.served.contract.name, namespace, this); + } + + /** + * A runtime failure of the live attempt `attempt`: `broken` (a `migrate` failing on access, a stored version newer + * than the code, a refused commit) or `transient` (a followed runtime failed, R4): the attempt is sealed and the + * object `failed` with `cause`; calls reject with it, holders are told, the state members keep their values once the + * admitted commits adopted, the natives close in the background (then its references are released), `onError` gets + * `object_failed` once. Documents and job runs stay. Recovery: a broken failure by `ref.retry()`, a transient one + * after its identity's backoff too (a new attempt, whose members the member sources follow once it is live). A + * failure of a sealed attempt is ignored, so several at once fail it once. + */ + failLive(attempt: Attempt, cause: FailureCause): void { const state = this.#state; if (state.kind !== "live" || state.attempt !== attempt) return; - const cause: FailureCause = { kind: "broken", error }; - this.host.backoff.delete(this.identity); - const failure: Failed = { kind: "failed", cause, retryAt: undefined }; + const { error } = cause; + if (cause.kind === "transient") this.host.backoff.failed(this.identity); + else this.host.backoff.delete(this.identity); + const failure: Failed = { kind: "failed", cause, retryAt: this.host.backoff.until(this.identity) }; this.#state = { kind: "failed", failure, holding: undefined }; this.#lastFailure = error; this.#outcome = settlement(); @@ -382,6 +399,26 @@ export class ChordObject implements Target { this.#idle(); } + /** A held state change must be stored by `deadline`: the timer is set for the earliest one. */ + #heldUntil(deadline: number): void { + if (this.#flush !== undefined && this.#flush.deadline <= deadline) return; + if (this.#flush !== undefined) clearTimeout(this.#flush.timer); + // A foreground timer: what is held is stored before the process may exit. + const timer = setTimeout(() => void this.#flushHeld(), Math.max(0, deadline - Date.now())); + this.#flush = { deadline, timer }; + } + + /** The timer fired: every held change past its deadline is stored, and the timer set for the next one. */ + async #flushHeld(): Promise { + this.#flush = undefined; + if (this.#state.kind === "ended") return; + const now = Date.now(); + const jobs = [...this.#jobs.values()]; + await Promise.all(jobs.map((records) => records.flushHeld(now))); + const next = Math.min(...jobs.map((records) => records.nextDeadline())); + if (Number.isFinite(next)) this.#heldUntil(next); + } + /** * Adds `delta` to its running and finishing job runs (and `host.activity.jobs`). A new run keeps a lingering object; * the last one settling on an object without references ends it as its last release would (`#idle`). @@ -580,6 +617,8 @@ export class ChordObject implements Target { const previous = this.host.endingOf(this.identity); this.#state = { kind: "ended", ended: { kind: "ended", error } }; for (const records of this.#jobs.values()) records.end(error); + if (this.#flush !== undefined) clearTimeout(this.#flush.timer); + this.#flush = undefined; this.documents.seal(); this.#stopLingering(); this.host.forget(this); @@ -648,6 +687,7 @@ export class ChordObject implements Target { if (teardown.kind !== "end") teardown.attempt.releaseReferences(); await this.documents.settled(); this.documents.end(); + this.host.store?.claims.release(this); for (const records of this.#jobs.values()) records.drop(); this.#jobs.clear(); return failures; diff --git a/packages/chord/src/next/host/routines.ts b/packages/chord/src/next/host/routines.ts index 824e0ee38..ea924ec7b 100644 --- a/packages/chord/src/next/host/routines.ts +++ b/packages/chord/src/next/host/routines.ts @@ -306,6 +306,7 @@ class OnRoutine { if (call.kind === "value") done(true); }, end: () => done(false), + endAfterDelivery: () => done(false), }; const aborted = (): void => done(false); signal?.addEventListener("abort", aborted, { once: true }); diff --git a/packages/chord/src/next/index.ts b/packages/chord/src/next/index.ts new file mode 100644 index 000000000..8e759f345 --- /dev/null +++ b/packages/chord/src/next/index.ts @@ -0,0 +1,50 @@ +// Chord v2 (`@earendil-works/chord/next`, experimental): every layer's API for apps, and the store's surface for +// adapters (Durable's `StoreStorage`), marked internal. + +export * from "./contracts/index.ts"; +export * from "./data/index.ts"; +export * from "./documents/index.ts"; +export * from "./host/index.ts"; +export * from "./jobs/index.ts"; +export * from "./link/index.ts"; +export * from "./state/index.ts"; +/** @internal The store's surface for adapters (slice 6b §2.1). */ +export { type DecodedRow, OwnerTable, StoreOwnerTables } from "./store/adapter.ts"; +/** @internal The sync layer's batch apply (slice 6b §1.6). */ +export { applyBatch } from "./store/apply-batch.ts"; +/** @internal */ +export type { Claimant } from "./store/claims.ts"; +/** @internal */ +export { + type DocumentAddress, + type DocumentContent, + type DocumentEntry, + type DocumentHistory, + type DocumentPoint, + type DocumentRecord, + type DocumentScope, + findDocument, + findDocumentEntry, + insertDocument, + listDocumentEntries, + listDocuments, + loadContent, + loadDocument, + type OwnedDocumentEntry, + readDocument, + readDocumentEntry, + retireRecord, + type StoredDocument, + writeContent, +} from "./store/engine.ts"; +/** @internal */ +export type { WhereNode } from "./store/expression.ts"; +export * from "./store/index.ts"; +/** @internal */ +export type { EncodedRow } from "./store/rows.ts"; +/** @internal */ +export type { Order } from "./store/sql.ts"; +/** @internal */ +export type { StagedWrite, Step } from "./store/step.ts"; +/** @internal */ +export type { ChordStore } from "./store/store.ts"; diff --git a/packages/chord/src/next/jobs/README.md b/packages/chord/src/next/jobs/README.md index 3fba3ca35..13594c9c4 100644 --- a/packages/chord/src/next/jobs/README.md +++ b/packages/chord/src/next/jobs/README.md @@ -105,8 +105,18 @@ host.serve(ChatStore, (_, self) => { only when it records something (`Transaction.staged` reads what it staged so far). - `JobOwner`: what a job needs of its object: label, host, caller, the object's context (without an abort signal), documents, `admit` (the drain) and `countRuns` (keep-alive and `host.activity.jobs`). -- Inside: `JobRun` (one run: record, phase, abort signal, invocation fence closed as `run` settles, `running` entry, - the `Run` handle over its detachable waiters), the +- Inside: `JobRun` (one run: record, phase, its current `Invocation`, `running` entry, the `Run` handle over its + detachable waiters; an `Invocation` is one call of `run` with its own seal token, abort signal and join, captured by + its ctx and its settling, so an invocation a seal replaced never writes or records for the next one), the run states (a keyed `DocumentHandle` by run id, apart from `self.state`'s, never drafted while absent) and `Running` (`running`'s list, copied per change; while watched it records the ops of each change, a run's state ops re-pathed under its entry, delivered as one frame per batch). + +Persisted jobs (slice 6b): `persist: { resume: "rerun" | "abandon" }` in a `local` type with empty params. Each +unsettled run's record (`stored.ts`) is a document in `chord.jobs`, its state one in `chord.job-states` +(`persisted.ts`, `run-states.ts`); a run is reserved (`running`) before `run`, counted after a crash (five in a row end it +`restart_limit`), written back `stopped` by the close writer after the drain, resumed on the next live attempt (per +`resume`); the records load once per object, again only after a load failed, a record of another version or fingerprint +blocked before its state is decoded; `abort(id, ctx)` is stored and joins the current invocation; +`ctx.state.change(fn, ctx, { persistence: { kind: "deferred" } })` holds a change (its ops folded in a `PendingOps`) until +the next immediate one, the outcome, the close writer or its deadline (one timer per object). diff --git a/packages/chord/src/next/jobs/definition.ts b/packages/chord/src/next/jobs/definition.ts index c0dc86082..852d4e80d 100644 --- a/packages/chord/src/next/jobs/definition.ts +++ b/packages/chord/src/next/jobs/definition.ts @@ -2,7 +2,7 @@ import { checkKeys, entriesOf } from "../contracts/declare.ts"; import type { Static, TSchema } from "../contracts/index.ts"; -import { checkName } from "../contracts/internal.ts"; +import { checkName, hash64 } from "../contracts/internal.ts"; import { type Snapshot, snapshotArguments, snapshotSchema } from "../contracts/snapshot.ts"; import type { ChordError, PluginError } from "../data/index.ts"; import type { JsonObject, StateDefinition } from "../documents/index.ts"; @@ -11,7 +11,7 @@ import type { JsonObject, StateDefinition } from "../documents/index.ts"; export type JobOptions< A extends TSchema, R extends TSchema, - D extends StateDefinition | undefined, + D extends StateDefinition | undefined, > = { /** The job's version; every change of its args, result or state bumps it. */ readonly version: number; @@ -19,23 +19,33 @@ export type JobOptions< readonly args: A; /** What `run` returns; not validated. */ readonly result: R; - /** A singleton document definition: one document per run, `ctx.state` in `run`, `outcome.state` in `finish`. */ + /** + * A singleton document definition: one document per run, `ctx.state` in `run`, `outcome.state` in `finish`; a + * persisted job's is persisted (`persist: true`, its `checkpointWhen`). + */ readonly state?: D; /** At most this many runs running or finishing per object; a start beyond it rejects `unavailable/busy`. */ readonly concurrency?: number; /** Per older version, how its args and result map forwards (used from slice 7 on; checked now). */ readonly accepts?: JobAccepts; - /** How a persisted run resumes after a crash; refused before slice 6. */ - readonly persist?: { readonly resume: "rerun" | "abandon" }; + /** + * Persists the runs in the object's store (slice 6b; only in `local` types with empty params, P3): how a run cut + * off by a crash or a stop resumes, `rerun` (its `run` again: idempotent work) or `abandon` (`finish` gets `failed { + * aborted, reason: "restart" }`). Its result is JSON only (no `bytes()`). + */ + readonly persist?: JobPersistence; }; +/** How a persisted job's runs resume after a crash or a stop. */ +export type JobPersistence = { readonly resume: "rerun" | "abandon" }; + /** Per older version, its mappings forwards: its args to the current args, its result to the current result. */ export type JobAccepts = { readonly [version: number]: { readonly args: (old: never) => Static; readonly result: (old: never) => Static }; }; /** The value of a job's state document: the definition's document, `undefined` without one. */ -type StateOf = D extends StateDefinition ? T : undefined; +type StateOf = D extends StateDefinition ? T : undefined; /** What a run ended with, as `finish` and `outcome` get it: its args and state, and its result or its error. */ export type Outcome = { readonly id: string; readonly args: A; readonly state: S } & ( @@ -45,18 +55,20 @@ export type Outcome = { readonly id: string; readonly args: A; readonly /** * Why a run failed: `run` threw (reported as `job_failed`; anything but a ChordError or PluginError becomes a - * PluginError `internal` with its message), or it was aborted (`abort`: `abort(id)`; `reload` and `restart` come with - * slices 7 and 6). + * PluginError `internal` with its message), it was aborted (`abort`: `abort(id)`; `restart`: cut off by its attempt's + * seal while the object continues, or a persisted run abandoned after a crash; `reload` comes with slice 7), or a + * persisted run was cut off five times in a row (`restart_limit`, D6). */ export type JobError = | { readonly kind: "threw"; readonly error: ChordError | PluginError } - | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" }; + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; /** Declares a job: owned work a call starts that outlives it (§15). */ export function defineJob< const A extends TSchema, const R extends TSchema, - D extends StateDefinition | undefined = undefined, + D extends StateDefinition | undefined = undefined, >(name: string, options: JobOptions): JobDefinition, Static, StateOf> { // Declaration policy: JavaScript callers reach every check below. checkName(name, "A job name"); @@ -67,12 +79,32 @@ export function defineJob< if (args.bytes) throw new Error(`${name}.args: job arguments are JSON (running carries them); no bytes()`); const result = snapshotSchema(options.result, `${name}.result`); if (state?.keyed) throw new Error(`${name}.state: expected a singleton document definition`); + if (persist !== undefined) checkPersistence(persist, state, result, name); if (concurrency !== undefined && !(Number.isSafeInteger(concurrency) && concurrency > 0)) { throw new Error(`${name}.concurrency: expected a positive integer`); } if (accepts !== undefined) checkJobAccepts(accepts, version, `${name}.accepts`); - if (persist !== undefined) throw new Error(`${name}.persist: persisted jobs are not supported before slice 6`); - return new JobDefinition(name, version, args, result, state, concurrency, accepts ?? {}); + const fingerprint = hash64( + JSON.stringify({ args: args.normal, result: result.normal, state: state?.fingerprint ?? null }), + ); + return new JobDefinition(name, version, args, result, state, concurrency, accepts ?? {}, persist, fingerprint); +} + +/** A persisted job's declaration: a resume rule, a persisted state document (if any), a JSON-only result. */ +function checkPersistence( + persist: unknown, + state: StateDefinition | undefined, + result: Snapshot, + name: string, +): void { + checkKeys(persist, ["resume"], `${name}.persist`); + const { resume } = persist as { readonly resume?: unknown }; + if (resume !== "rerun" && resume !== "abandon") + throw new Error(`${name}.persist.resume: expected "rerun" or "abandon"`); + if (state !== undefined && !state.persist) { + throw new Error(`${name}.state: a persisted job's state is a persisted document (persist: true)`); + } + if (result.bytes) throw new Error(`${name}.result: a persisted job's result is JSON (it is stored); no bytes()`); } /** Checks an accepts table's shape: older versions only, each with an `args` and a `result` function. */ @@ -96,10 +128,14 @@ export class JobDefinition { /** The args schema, Chord's copy; its compiled check is cached by this snapshot. */ readonly args: Snapshot; readonly result: Snapshot; - readonly state: StateDefinition | undefined; + readonly state: StateDefinition | undefined; readonly concurrency: number | undefined; /** The older versions' mappings, checked at declaration; slice 7 maps older runs with them. */ readonly accepts: JobAccepts; + /** How its runs persist; `undefined` for a job in memory. */ + readonly persist: JobPersistence | undefined; + /** `hash64` of its args', result's and state's normal forms: a stored run of another fingerprint is blocked. */ + readonly fingerprint: string; /** Type only: the args, result and state types. */ declare readonly "~types"?: { readonly args: A; readonly result: R; readonly state: S }; @@ -108,9 +144,11 @@ export class JobDefinition { version: number, args: Snapshot, result: Snapshot, - state: StateDefinition | undefined, + state: StateDefinition | undefined, concurrency: number | undefined, accepts: JobAccepts, + persist: JobPersistence | undefined, + fingerprint: string, ) { this.name = name; this.version = version; @@ -119,5 +157,7 @@ export class JobDefinition { this.state = state; this.concurrency = concurrency; this.accepts = accepts; + this.persist = persist; + this.fingerprint = fingerprint; } } diff --git a/packages/chord/src/next/jobs/index.ts b/packages/chord/src/next/jobs/index.ts index 5a8cd4739..b2472d220 100644 --- a/packages/chord/src/next/jobs/index.ts +++ b/packages/chord/src/next/jobs/index.ts @@ -9,5 +9,5 @@ export { type Outcome, } from "./definition.ts"; export type { JobCtx, JobHandle, JobImplementation, RunState, StartOptions } from "./job.ts"; -export type { Run } from "./run.ts"; +export type { BlockedReason, Run } from "./run.ts"; export type { RunningEntry } from "./running.ts"; diff --git a/packages/chord/src/next/jobs/job.ts b/packages/chord/src/next/jobs/job.ts index 12e4b7403..70d6703fd 100644 --- a/packages/chord/src/next/jobs/job.ts +++ b/packages/chord/src/next/jobs/job.ts @@ -7,13 +7,13 @@ import type { Draft } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import type { Watchable } from "../../watch.ts"; import { ChordError, isChordError, PluginError, type Principal } from "../data/index.ts"; -import type { JsonObject, Tx } from "../documents/index.ts"; +import type { CommitPersistence, JsonObject, Tx } from "../documents/index.ts"; import type { Documents, SealToken, Transaction } from "../documents/internal.ts"; import { type Ctx, sleep } from "../host/index.ts"; import { type AnyHost, CallContext, type Caller, nextDelay } from "../host/internal.ts"; import type { Outcome } from "./definition.ts"; import type { JobRecords, Settled } from "./records.ts"; -import type { AnyOutcome, JobRun, Run } from "./run.ts"; +import type { AnyOutcome, Invocation, JobRun, Run } from "./run.ts"; import type { RunningEntry } from "./running.ts"; /** A job's handle, `self.job(Job, { run, finish? })`; `A`, `R` and `S` are its args, result and state. */ @@ -27,10 +27,12 @@ export interface JobHandle { /** Waits for the outcome of the unsettled run `id`; a settled or unknown id rejects `gone`. */ outcome(id: string, context: Context): Promise>; /** - * Fires a running run's abort signal; drops a blocked run (no `finish`; its waiters reject `aborted`); ignored for - * finishing runs and unknown ids. + * Aborts run `id` (D7, as Durable's `abortTask`): marks it (stored for a persisted job) and fires its `run`'s + * signal, resolving `"marked"` once its `run` settled (a run not invoked yet ends `aborted` without one); a + * finishing run is only marked, a blocked one dropped (no `finish`; its waiters reject `aborted`). `"terminal"` when + * no unsettled run has that id. `context` ends only this wait. */ - abort(id: string): void; + abort(id: string, context: Context): Promise<"marked" | "terminal">; /** The unsettled runs, one entry each in start order, tagged by phase (a JSON watchable, a state member). */ readonly running: Watchable[]>; } @@ -60,9 +62,15 @@ export interface RunState { readonly value: S; /** * One transaction on the object's line drafting only this document, under `context` without its abort signal: - * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. + * admitted while `run` runs (also after an abort, with the run's own ctx), refused `released` once it settled. A + * persisted job's change may be `deferred` (D8): adopted and published at once, stored within `withinMs` (1,000 ms by + * default) or earlier; a crash loses it. In memory the option changes nothing. */ - change(fn: (draft: Draft) => T, context: Context): Promise; + change( + fn: (draft: Draft) => T, + context: Context, + options?: { readonly persistence?: CommitPersistence }, + ): Promise; } /** Where a job's handle runs its work: its attempt, as the job sees it. */ @@ -105,6 +113,8 @@ export class Job implements JobHandle #state: "declared" | "live" | "sealed" = "declared"; /** Runs whose `run` runs on this attempt. */ readonly #invocations = new Set(); + /** The seal's error, once sealed: a reservation committing after it cuts its run off with it. */ + #sealed: ChordError | undefined; /** Runs whose finish runs on this attempt. */ readonly #finishing = new Set(); @@ -128,8 +138,18 @@ export class Job implements JobHandle return this.#records.outcome(id, context); } - abort(id: string): void { - this.#records.abort(id); + abort(id: string, context: Context): Promise<"marked" | "terminal"> { + return this.#records.abort(id, context); + } + + /** A persisted job's runs, loaded once per object at its first activation (before it goes live). */ + load(context: Context): Promise { + return this.#records.load(context); + } + + /** The close writer of this job's runs, after the attempt's drain (slice 6b §4.2). */ + writeAtClose(context: Context): Promise { + return this.#records.writeAtClose(context); } /** The attempt went live: starts are taken, and the records' waiting runs are invoked or finished here. */ @@ -170,6 +190,7 @@ export class Job implements JobHandle seal(error: ChordError): void { if (this.#state === "sealed") return; this.#state = "sealed"; + this.#sealed = error; this.#records.detach(this, this.#finishing); this.#finishing.clear(); for (const run of this.#invocations) this.#records.cut(run, error); @@ -177,30 +198,70 @@ export class Job implements JobHandle } /** - * Runs `run` for a committed run (with its signal aborted already after an early abort). The invocation and the + * Runs `run` for a committed run (with its signal aborted already after an early abort); a persisted run's + * reservation commits first (`running`: a crash from then on counts, D6). The reservation, the invocation and the * run's finish are one piece of work in the drain, admitted before plugin code runs: `run` may end its own object * before its first await. */ invoke(run: JobRun): void { + const invocation = run.begin(); + if (!this.#records.persisted || run.phase === "running") { + this.#start(run, invocation); + return; + } + const { documents, token, context } = this.#owner; + const reserving = documents + .tx((tx) => this.#records.reserve(tx, run), token, context) + .then( + () => { + if (this.#state !== "sealed") { + this.#start(run, invocation); + return; + } + this.#records.cut(run, this.#sealed!); + // No `run` started: nothing else ends it, and an abort joined to it returns. + invocation.ended(); + }, + () => { + invocation.settle(); + invocation.ended(); + // Refused because this attempt was sealed (at admission, or queued past the seal): the run is pending still, + // for the next live attempt (this one is detached already). Any other failure (the store closed or failed, + // the object ended) is not retried here: retrying on this attempt would only fail again. + if (this.#state === "sealed") this.#records.requeue(run); + }, + ); + this.#owner.admit(reserving); + } + + /** Starts `invocation` of `run` on this attempt: its ctx and its settling act for this invocation only. */ + #start(run: JobRun, invocation: Invocation): void { const { context, caller, host } = this.#owner; - const ctx = new JobContext(withAbortSignal(run.abort.signal, context), caller, host, run.id, this.#stateOf(run)); + const signal = invocation.abort.signal; + const ctx = new JobContext( + withAbortSignal(signal, context), + caller, + host, + run.id, + this.#stateOf(run, invocation), + ); const work = Promise.withResolvers(); this.#owner.admit(work.promise, () => ctx.settle()); this.#invocations.add(run); - run.invocation = "running"; - const invocation = (async () => { + const running = (async () => { try { return await this.#implementation.run(run.args, ctx); } finally { // The fence closes as `run` settles: a change made later is refused. - run.invocation = "settled"; + invocation.settle(); } })(); - invocation + running .then( - (result) => this.#settled(run, ctx, { kind: "done", result }), - (error: unknown) => this.#settled(run, ctx, this.#failed(run, error)), + (result) => this.#settled(run, invocation, ctx, { kind: "done", result }), + (error: unknown) => this.#settled(run, invocation, ctx, this.#failed(run, invocation, error)), ) + .finally(() => invocation.ended()) .then(work.resolve); } @@ -214,21 +275,22 @@ export class Job implements JobHandle this.#owner.admit(work); } - /** How a throwing `run` ended: `aborted` after its signal fired, otherwise `threw` (reported). */ - #failed(run: JobRun, error: unknown): Settled { - if (run.abort.signal.aborted) return { kind: "failed", error: { kind: "aborted", reason: "abort" } }; + /** How a throwing `run` ended: `aborted` after its invocation's signal fired, otherwise `threw` (reported). */ + #failed(run: JobRun, invocation: Invocation, error: unknown): Settled { + if (invocation.abort.signal.aborted) return { kind: "failed", error: { kind: "aborted", reason: "abort" } }; this.#records.report({ kind: "job_failed", id: run.id, error }); const thrown = isChordError(error) || error instanceof PluginError ? error : internal(error); return { kind: "failed", error: { kind: "threw", error: thrown } }; } /** - * `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut it off (which - * recorded its own), then the run finishes here while this attempt is live. + * `invocation` of `run` settled: its `ctx.get` references go; its outcome is recorded unless the attempt's seal cut + * it off (which recorded its own, or stopped the run for a later invocation), then the run finishes here while this + * attempt is live. */ - async #settled(run: JobRun, ctx: JobContext, ended: Settled): Promise { + async #settled(run: JobRun, invocation: Invocation, ctx: JobContext, ended: Settled): Promise { ctx.settle(); - if (run.token.sealed) return; + if (invocation.token.sealed || run.invocation !== invocation) return; this.#invocations.delete(run); this.#records.settled(run, ended); } @@ -238,14 +300,14 @@ export class Job implements JobHandle this.#finishing.add(run); let delay: number | undefined; try { - for (let failures = 1; ; failures++) { + for (;;) { try { return await documents.tx( async (tx) => { const staging = this.#records.staging(tx); staging.finishing = run; await this.#implementation.finish?.(run.outcome!, tx); - staging.changes.push({ kind: "delete", run }); + this.#records.change(tx, { kind: "delete", run }); }, token, context, @@ -253,7 +315,12 @@ export class Job implements JobHandle } catch (error) { if (this.#state === "sealed") return; this.#records.report({ kind: "job_finish_failed", id: run.id, error }); - if (failures === FINISH_ATTEMPTS) return await this.#records.record(run, "block"); + // The count is the run's (stored for a persisted job): failures count across a crash. + const finishFailures = run.finishFailures + 1; + if (finishFailures >= FINISH_ATTEMPTS) { + return await this.#records.record(run, { kind: "block", run, reason: "finish_failed" }); + } + await this.#records.record(run, { kind: "update", run, fields: { finishFailures } }); delay = nextDelay(delay); try { await sleep(delay, withAbortSignal(signal, context)); @@ -267,8 +334,8 @@ export class Job implements JobHandle } } - /** `ctx.state` of `run`, when the job has a state: changes under the run's own token while `run` runs. */ - #stateOf(run: JobRun): RunState | undefined { + /** `ctx.state` of `invocation` of `run`, when the job has a state: changes under its token while it runs. */ + #stateOf(run: JobRun, invocation: Invocation): RunState | undefined { const records = this.#records; const states = records.states; if (states === undefined) return undefined; @@ -285,16 +352,59 @@ export class Job implements JobHandle if (value === undefined) throw refused(); return value; }, - change: (fn: (draft: Draft) => T, context: Context): Promise => { - if (run.invocation !== "running") return Promise.reject(refused()); + change: ( + fn: (draft: Draft) => T, + context: Context, + options: { readonly persistence?: CommitPersistence } = {}, + ): Promise => { + if (invocation.state !== "running") return Promise.reject(refused()); + let persistence: CommitPersistence; + try { + persistence = checkedPersistence(options.persistence, records.name); + } catch (error) { + return Promise.reject(error); + } // Never the abort signal: a run records its progress also after an abort. const unaborted = withoutAbortSignal(context); - return documents.tx(async (tx) => fn(await tx.draftSlot(states, run.id)), run.token, unaborted); + // A change resetting the restart count is stored at once: the reset must be durable (D8). + const now = run.restarts === 0 ? persistence : IMMEDIATE; + return documents.tx( + async (tx) => { + const value = fn(await tx.draftSlot(states, run.id)); + records.progressed(tx, run); + return value; + }, + invocation.token, + unaborted, + now, + ); }, }; } } +const IMMEDIATE: CommitPersistence = { kind: "immediate" }; + +/** + * A state change's `persistence` (a trust boundary: plugin code): `immediate` when omitted; a deferred one's `withinMs` + * a number from 0 (`Infinity` waits for the others), `invalid/args` otherwise. + */ +function checkedPersistence(persistence: CommitPersistence | undefined, what: string): CommitPersistence { + if (persistence === undefined || persistence.kind === "immediate") return IMMEDIATE; + const { withinMs } = persistence; + if ( + persistence.kind !== "deferred" || + (withinMs !== undefined && !(typeof withinMs === "number" && withinMs >= 0)) + ) { + throw new ChordError({ + code: "invalid", + reason: "args", + message: `${what}: persistence is immediate or deferred with withinMs >= 0`, + }); + } + return persistence; +} + /** A plugin's throw that is neither a ChordError nor a PluginError, as a run's outcome keeps it. */ function internal(error: unknown): PluginError { return new PluginError("internal", error instanceof Error ? error.message : String(error)); diff --git a/packages/chord/src/next/jobs/persisted.ts b/packages/chord/src/next/jobs/persisted.ts new file mode 100644 index 000000000..9802ad441 --- /dev/null +++ b/packages/chord/src/next/jobs/persisted.ts @@ -0,0 +1,100 @@ +// A persisted job's run records in its object's store (slice 6b §4.1, D5): the record document of each unsettled run +// (namespace `chord.jobs`, kind the job name, key the run id, `history: none`), written whole in the transactions that +// change the run, and read once per object when it first activates; for a run of another job version, also what is +// stored of its state document (`chord.job-states`), never decoded, so its drop retires it whatever the code declares. + +import type { Context } from "../../context/index.ts"; +import type { JsonObject } from "../documents/index.ts"; +import type { Persistence, Stored } from "../documents/internal.ts"; +import { changeDocument, createDocument, retireDocument } from "../store/engine.ts"; +import type { StagedWrite } from "../store/step.ts"; +import { RUN_RECORD, recordOf, type StoredRun } from "./stored.ts"; + +/** A run record as a load found it: its run id, the record and what is stored of it. */ +export type LoadedRecord = { readonly id: string; readonly record: StoredRun; readonly stored: Stored }; + +/** The keys a load reads per page. */ +const KEYS_PAGE = 100; + +/** The record documents of one persisted job's runs, and what is stored of each. */ +export class PersistedRuns { + readonly #records: Persistence; + readonly #states: Persistence; + readonly #stored = new Map(); + /** What is stored of the state documents of runs of another job version, by run id. */ + readonly #blockedStates = new Map(); + + /** `records` and `states` are the job's record and state documents' store sides (`chord.jobs`, `chord.job-states`). */ + constructor(records: Persistence, states: Persistence) { + this.#records = records; + this.#states = states; + } + + /** + * Stages run `id`'s record `record` into `writes` (rule 8: serialized now): its creation, or a new base of what is + * stored; returns the adoption, which takes what the commit stored. + */ + stage(id: string, record: StoredRun, writes: StagedWrite[]): () => void { + const text = JSON.stringify(record); + const stored = this.#stored.get(id); + const { version, fingerprint } = RUN_RECORD; + const value = record as unknown as JsonObject; + if (stored === undefined) { + const created = { id: 0 }; + writes.push( + createDocument(this.#records.at(id), { version, fingerprint, value }, text, (made) => { + created.id = made; + }), + ); + return () => this.#stored.set(id, { id: created.id, revision: 1, version, deltasSinceBase: 0 }); + } + writes.push(changeDocument(stored.id, stored.revision, { kind: "base", version, fingerprint, value }, text)); + return () => this.#stored.set(id, { ...stored, revision: stored.revision + 1 }); + } + + /** + * Stages the retirement of run `id`'s record into `writes` (the run settled or was dropped), and of its state + * document when the run is of another job version; the adoption forgets them. + */ + stageRetire(id: string, writes: StagedWrite[]): () => void { + const stored = this.#stored.get(id); + if (stored !== undefined) writes.push(retireDocument(stored.id, stored.revision)); + const state = this.#blockedStates.get(id); + if (state !== undefined) writes.push(retireDocument(state.id, state.revision)); + return () => { + this.#stored.delete(id); + this.#blockedStates.delete(id); + }; + } + + /** Finds what is stored of run `id`'s state document (a run of another job version), keeping it for the drop. */ + async findBlockedState(id: string, context: Context): Promise { + const stored = await this.#states.find(id, context); + if (stored !== undefined) this.#blockedStates.set(id, stored); + } + + /** Every stored record of the job, in run id order, page by page with plain reads (activation holds the line). */ + async load(context: Context): Promise { + const loaded: LoadedRecord[] = []; + let after: string | undefined; + for (;;) { + const keys = await this.#records.keys(after, KEYS_PAGE, context); + for (const id of keys) { + const document = await this.#records.read(id, context); + if (document === undefined) continue; + const { record: stored, value, deltasSinceBase } = document; + const found = { id: stored.id, revision: stored.revision, version: stored.version, deltasSinceBase }; + this.#stored.set(id, found); + loaded.push({ id, record: recordOf(value as JsonObject), stored: found }); + } + if (keys.length < KEYS_PAGE) return loaded; + after = keys.at(-1); + } + } + + /** The object's documents are dropped: nothing is known of the store any more. */ + drop(): void { + this.#stored.clear(); + this.#blockedStates.clear(); + } +} diff --git a/packages/chord/src/next/jobs/records.ts b/packages/chord/src/next/jobs/records.ts index b1e8e306b..73cc9f91e 100644 --- a/packages/chord/src/next/jobs/records.ts +++ b/packages/chord/src/next/jobs/records.ts @@ -1,20 +1,27 @@ -// A job's records on its object (§15, slice 6a §1.2): its runs by id and by key, the active count, the run state -// documents, `running` and the runs waiting for a live attempt, across the object's attempts. Records and state -// documents change only in transactions on the object's line (staged changes adopted in the commit step). +// A job's records on its object (§15, slice 6a §1.2, slice 6b §4): its runs by id and by key, the active count, the +// run state documents, `running` and the runs waiting for a live attempt, across the object's attempts. Records and +// state documents change only in transactions on the object's line (staged changes adopted in the commit step); a +// persisted job's also in its object's store, in the same commits, loaded once per object, and written back stopped by +// the close writer when a clean stop cuts its runs off. -import type { Context } from "../../context/index.ts"; -import { type Op, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { type Context, withoutAbortSignal } from "../../context/index.ts"; +import type { Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import { ChordError } from "../data/index.ts"; import { check, validatorFor } from "../data/internal.ts"; -import type { JsonObject, StateDefinition } from "../documents/index.ts"; -import type { DocumentSlot, Documents, Staged, Transaction } from "../documents/internal.ts"; +import type { JsonObject } from "../documents/index.ts"; +import { type Documents, Persistence, type Staged, type Transaction } from "../documents/internal.ts"; import type { AnyHost } from "../host/internal.ts"; import { uuidv7 } from "../host/internal.ts"; +import type { StagedWrite } from "../store/step.ts"; +import type { ChordStore } from "../store/store.ts"; import type { JobDefinition, JobError } from "./definition.ts"; import type { Job, StartOptions } from "./job.ts"; -import { type AnyOutcome, JobRun, type Run } from "./run.ts"; +import { type LoadedRecord, PersistedRuns } from "./persisted.ts"; +import { type AnyOutcome, type BlockedReason, JobRun, type Phase, type Run } from "./run.ts"; +import { RunStates } from "./run-states.ts"; import { Running } from "./running.ts"; +import { outcomeOf, type StoredFields, storedRun } from "./stored.ts"; /** What a job's records need of their object. */ export type RecordsOwner = { @@ -23,8 +30,12 @@ export type RecordsOwner = { /** The object's context, without an abort signal: outcomes are recorded under it. */ readonly context: Context; readonly documents: Documents; + /** The host's store and the object's owner name in it, when the object may persist (P3). */ + readonly store: { readonly store: ChordStore; readonly owner: string } | undefined; /** Counts running and finishing runs: they keep the object alive (`host.activity.jobs`). */ countRuns(delta: number): void; + /** A held state change must be stored by `deadline`: the object's one timer flushes it (D8). */ + held(deadline: number): void; }; /** How a run's invocation ended, before its outcome is recorded. */ @@ -32,21 +43,45 @@ export type Settled = | { readonly kind: "done"; readonly result: unknown } | { readonly kind: "failed"; readonly error: JobError }; +/** What a change sets on a run's record besides its phase transitions below (D6, D7, the finish failure count). */ +type RunFields = { + readonly phase?: Phase; + readonly restarts?: number; + readonly finishFailures?: number; + readonly abortMarked?: true; +}; + +/** A loaded run and the change its activation writes, if any. */ +type Restored = { readonly run: JobRun; readonly change: RecordChange | undefined }; + /** A change of the records, staged in a transaction and adopted in its commit step. */ type RecordChange = - | { readonly kind: "create"; readonly run: JobRun; readonly state: JsonObject | undefined } + | { readonly kind: "create"; readonly run: JobRun } + | { readonly kind: "update"; readonly run: JobRun; readonly fields: RunFields } | { readonly kind: "outcome"; readonly run: JobRun; readonly outcome: AnyOutcome } - | { readonly kind: "delete" | "block" | "drop"; readonly run: JobRun }; + | { readonly kind: "block"; readonly run: JobRun; readonly reason: BlockedReason } + | { readonly kind: "delete"; readonly run: JobRun } + | { readonly kind: "drop"; readonly run: JobRun }; + +/** What a commit step adopts of the records: a change, or a loaded run taken into memory as stored. */ +type Adopted = RecordChange | { readonly kind: "restore"; readonly run: JobRun }; /** One transaction's changes of one job: the runs it started (dedupe, concurrency) and whose finish it is. */ export type Staging = Staged & { readonly started: JobRun[]; - readonly changes: RecordChange[]; + readonly changes: Adopted[]; + /** What the transaction's store writes leave in memory, adopted with the changes. */ + readonly adopts: (() => void)[]; finishing: JobRun | undefined; }; const RESTART: Settled = { kind: "failed", error: { kind: "aborted", reason: "restart" } }; +const ABORTED: Settled = { kind: "failed", error: { kind: "aborted", reason: "abort" } }; + +/** A persisted run cut off this many times in a row ends `restart_limit` (D6, Durable's limit). */ +const RESTART_LIMIT = 5; + /** * The records of one job name on one object. The live attempt's `Job` invokes and finishes its runs; between live * attempts, committed runs not yet invoked and runs whose outcome is recorded wait for the next one. @@ -56,6 +91,8 @@ export class JobRecords { readonly running: Running; readonly #owner: RecordsOwner; readonly #states: RunStates | undefined; + /** The store side of a persisted job's runs; `undefined` in memory. */ + readonly #persisted: PersistedRuns | undefined; readonly #runs = new Map(); readonly #keys = new Map(); /** Running and finishing runs: concurrency and keep-alive count them. */ @@ -64,17 +101,30 @@ export class JobRecords { #job: Job | undefined; /** Runs waiting for a live attempt: to be invoked (`invocation` pending) or finished (phase `finishing`). */ readonly #waiting = new Set(); + /** Runs a clean stop cut off whose record the close writer has not written `stopped` yet. */ + readonly #stopped = new Set(); + /** Whether a persisted job's records were loaded (once per object). */ + #loaded = false; /** The object's end error, once it ended. */ #ended: ChordError | undefined; constructor(definition: JobDefinition, owner: RecordsOwner) { this.definition = definition; this.#owner = owner; - const { state } = definition; - this.#states = - state === undefined + const { state, persist, name } = definition; + const store = persist === undefined ? undefined : owner.store!; + const states = { + documents: owner.documents, + changed: (id: string, ops: readonly Op[]) => this.#stateChanged(id, ops), + held: (deadline: number) => owner.held(deadline), + }; + const persistence = + store === undefined ? undefined : new Persistence(store.store, store.owner, "chord.job-states", name); + this.#states = state === undefined ? undefined : new RunStates(state, states, persistence); + this.#persisted = + store === undefined ? undefined - : new RunStates(state, owner.documents, (id, ops) => this.#stateChanged(id, ops)); + : new PersistedRuns(new Persistence(store.store, store.owner, "chord.jobs", name), persistence!); this.running = new Running(owner.documents.batch, this.name); } @@ -92,15 +142,24 @@ export class JobRecords { return run.handle.outcome(context); } - /** Fires a running run's signal; drops a blocked run; ignored for finishing runs, unknown ids and after the end. */ - abort(id: string): void { + /** + * Aborts run `id` (D7, Durable's `abortTask`): a blocked run is dropped; a finishing one is only marked; any other + * is marked (stored for a persisted job) in a transaction, its signal fired after the commit, outside the batch and + * the line, and the abort joins its invocation. `"marked"`, or `"terminal"` when no unsettled run has that id. + */ + async abort(id: string, context: Context): Promise<"marked" | "terminal"> { const run = this.#runs.get(id); - if (run === undefined || this.#ended !== undefined) return; - if (run.phase === "running") { - run.abort.abort(new ChordError({ code: "aborted", message: `${this.name}: run ${id} aborted` })); - } else if (run.phase === "blocked") { - void this.record(run, "drop"); + if (run === undefined || this.#ended !== undefined) return "terminal"; + if (run.phase === "blocked") { + await this.record(run, { kind: "drop", run }); + return "marked"; } + if (!run.abortMarked) await this.record(run, { kind: "update", run, fields: { abortMarked: true } }); + // A finishing run's outcome stays: it is only marked. + if (run.phase === "finishing") return "marked"; + run.invocation?.abort.abort(new ChordError({ code: "aborted", message: `${this.name}: run ${id} aborted` })); + await run.over(context); + return "marked"; } /** See `Job.stage`: the checks and the staged creation, against the object's runs. */ @@ -120,26 +179,62 @@ export class JobRecords { const message = `${this.name}: at its concurrency limit (${concurrency})`; throw new ChordError({ code: "unavailable", reason: "busy", message }); } - const state = definition.state?.initial(context); + const persisted = this.#persisted !== undefined; + const trace = persisted ? context.telemetry.propagation?.inject(context.telemetry) : undefined; // Checked above: the args are JSON (the definition refuses bytes). - const run = new JobRun(uuidv7(), definition.version, key, args as JsonValue); + const run = new JobRun( + uuidv7(), + definition.version, + key, + args as JsonValue, + persisted ? "pending" : "running", + trace, + ); const staging = this.staging(tx); staging.started.push(run); - staging.changes.push({ kind: "create", run, state }); + this.change(tx, { kind: "create", run }); + const state = definition.state?.initial(context); + if (state !== undefined) staging.adopts.push(this.#states!.stageCreate(run.id, state, () => this.#store(tx))); return run.handle; } + /** + * Stages `change` in `tx`: in the job's staging, adopted in the commit step, and for a persisted job the run's record + * written with it (rule 8: serialized now), and for a settled run its state document retired. + */ + change(tx: Transaction, change: RecordChange): void { + const staging = this.staging(tx); + staging.changes.push(change); + const { run } = change; + const states = this.#states; + const retires = change.kind === "delete" || change.kind === "drop"; + if (retires && states?.current(run.id) !== undefined) tx.retireSlot(states, run.id); + const persisted = this.#persisted; + if (persisted === undefined) return; + const writes = this.#store(tx); + if (retires) { + staging.adopts.push(persisted.stageRetire(run.id, writes)); + return; + } + const fields = recordFields(change); + staging.adopts.push(persisted.stage(run.id, storedRun(run, this.definition.fingerprint, fields), writes)); + } + /** This job's changes in `tx`, made on the first. */ staging(tx: Transaction): Staging { return tx.stage(this, () => { - const changes: RecordChange[] = []; + const changes: Adopted[] = []; + const adopts: (() => void)[] = []; return { started: [], changes, + adopts, finishing: undefined, adopt: () => { // A step admitted before the object ended adopts after it: the records are not changed any more. if (this.#ended !== undefined) return; + // What is stored first: a new run's state document exists when its `running` entry is made. + runAll(adopts); for (const change of changes) this.#adopt(change); }, committed: () => { @@ -158,6 +253,49 @@ export class JobRecords { }); } + /** + * A persisted job's runs, loaded once per object at its first activation (slice 6b §4.2), again only after a load + * failed (design §9): each record read; a record of another version or fingerprint blocked (`version`) as stored, + * its state document left undecoded; the others' state documents read and decoded, a run cut off while `running` + * counted (`restarts + 1`; at the limit it ends `restart_limit`, reported `job_interrupted`), `stopped` and + * `pending` ones made `pending` again. The runs and their activation changes are adopted in one commit, all before + * anything runs; a failure leaves nothing loaded. + */ + async load(context: Context): Promise { + const persisted = this.#persisted; + if (persisted === undefined || this.#loaded) return; + const restored: Restored[] = []; + try { + for (const loaded of await persisted.load(context)) restored.push(await this.#restored(loaded, context)); + if (restored.length > 0) { + const { documents } = this.#owner; + await documents.tx((tx) => this.#restoreAll(tx, restored), documents.own, context); + } + } catch (error) { + persisted.drop(); + for (const { run } of restored) this.#states?.retire(run.id); + throw error; + } + this.#loaded = true; + for (const { run, change } of restored) { + if ( + change?.kind === "outcome" && + change.outcome.kind === "failed" && + change.outcome.error.kind === "restart_limit" + ) { + this.report({ kind: "job_interrupted", id: run.id, restarts: change.outcome.error.restarts }); + } + } + for (const { run } of restored) this.#need(run); + } + + /** Stages the loaded runs in `tx`: each restored, then the activation changes (adopted after every restore). */ + #restoreAll(tx: Transaction, restored: readonly Restored[]): void { + const staging = this.staging(tx); + for (const { run } of restored) staging.changes.push({ kind: "restore", run }); + for (const { change } of restored) if (change !== undefined) this.change(tx, change); + } + /** `job`, of the attempt that went live, invokes and finishes the runs from now on; the waiting ones first. */ attach(job: Job): void { if (this.#ended !== undefined) return; @@ -176,6 +314,24 @@ export class JobRecords { for (const run of finishing) this.#need(run); } + /** + * A run whose reservation its sealed attempt never committed waits for the next live attempt (dispatched now when one + * is attached already: the sealed attempt was detached at its seal). + */ + requeue(run: JobRun): void { + this.#need(run); + } + + /** Records `run` running before its `run` is invoked (a persisted run's reservation: a crash after it counts). */ + reserve(tx: Transaction, run: JobRun): void { + this.change(tx, { kind: "update", run, fields: { phase: "running" } }); + } + + /** A state change of a run with restarts counted resets them in its commit (progress, D6). */ + progressed(tx: Transaction, run: JobRun): void { + if (run.restarts !== 0) this.change(tx, { kind: "update", run, fields: { restarts: 0 } }); + } + /** * A run's invocation settled: its outcome is recorded (phase `finishing`) in a transaction of the object behind the * state changes admitted before, so `outcome.state` has them; then it is finished on the live attempt. @@ -185,30 +341,38 @@ export class JobRecords { } /** - * A running run whose attempt was sealed while the object continues: its signal fires with `error`, its invocation - * is fenced (later state changes refused, the open one discarded, its settling ignored), and it ends `failed { - * aborted, reason: "restart" }`, finished on the next live attempt. + * A running run whose attempt was sealed: its signal fires with `error`, its invocation is fenced (later state + * changes refused, the open one discarded, its settling ignored). In memory it ends `failed { aborted, reason: + * "restart" }`, finished on the next live attempt; persisted it is stopped (R3), written back by the close writer and + * resumed by the next live attempt as after a clean close. */ cut(run: JobRun, error: ChordError): void { - run.invocation = "settled"; - this.#owner.documents.fence(run.token); - run.abort.abort(error); + // Only an invoked run is cut: its current invocation is the one the seal stops. + const invocation = run.invocation!; + invocation.settle(); + this.#owner.documents.fence(invocation.token); + invocation.abort.abort(error); + if (this.#persisted !== undefined) { + run.phase = "stopped"; + // Resumed per the job's `resume` (an `abandon` job's run ends `failed { aborted, restart }`). + run.resuming = true; + this.#stopped.add(run); + if (this.#ended === undefined) this.#need(run); + return; + } if (this.#ended === undefined) void this.#recordOutcome(run, RESTART); } /** - * Records `kind` for `run` in a transaction of the object: `block` a finishing run, `drop` a blocked one (its - * record, key and state document go), if it still is one there. The object's end ends it. + * Records `change` (a run's finish failure, its block, its drop) in a transaction of the object, if `run` is still + * one of its runs there. The object's end ends it. */ - async record(run: JobRun, kind: "block" | "drop"): Promise { + async record(run: JobRun, change: RecordChange): Promise { const { documents, context } = this.#owner; try { await documents.tx( (tx) => { - const current = this.#runs.get(run.id) === run; - if (current && run.phase === (kind === "block" ? "finishing" : "blocked")) { - this.staging(tx).changes.push({ kind, run }); - } + if (this.#runs.get(run.id) === run) this.change(tx, change); }, documents.own, context, @@ -218,19 +382,81 @@ export class JobRecords { } } + /** + * The close writer (slice 6b §4.2, Durable's `writeAtClose`): one commit past the seal writing the runs a clean stop + * cut off `stopped` and every held state change, straight to the store (the object's transactions are refused by + * now). Its failure is the store's; the records stay as they were (a crash counts them). + */ + async writeAtClose(context: Context): Promise { + const persisted = this.#persisted; + if (persisted === undefined) return; + const writes: StagedWrite[] = []; + const adopts: (() => void)[] = []; + for (const run of this.#stopped) { + adopts.push(persisted.stage(run.id, storedRun(run, this.definition.fingerprint), writes)); + } + this.#stopped.clear(); + for (const id of this.#states?.heldIds ?? []) { + const adopt = this.#states!.stageHeld(id, writes); + if (adopt !== undefined) adopts.push(adopt); + } + if (writes.length === 0) return; + const fence = { admit: () => true, adopt: () => runAll(adopts) }; + try { + await this.#owner.store!.store.commit(writes, fence, withoutAbortSignal(context)); + } catch { + // The store failed or refused it (reported as the store's): a crash loses as much. + } + } + + /** Stores every held state change whose deadline passed (`now`), in one transaction of the object (D8). */ + async flushHeld(now: number): Promise { + const states = this.#states; + if (states === undefined || states.heldIds.length === 0) return; + const { documents, context } = this.#owner; + try { + await documents.tx( + (tx) => { + const staging = this.staging(tx); + for (const id of states.heldIds) { + if (states.deadline(id) > now) continue; + const adopt = states.stageHeld(id, this.#store(tx)); + if (adopt !== undefined) staging.adopts.push(adopt); + } + }, + documents.own, + context, + ); + } catch { + // The object is ending: its close writer stores what is held. + } + } + + /** The earliest deadline of a held state change (`Infinity`: none). */ + nextDeadline(): number { + const states = this.#states; + if (states === undefined) return Number.POSITIVE_INFINITY; + return Math.min(Number.POSITIVE_INFINITY, ...states.heldIds.map((id) => states.deadline(id))); + } + /** The value of run `id`'s state document, `undefined` without one or once retired. */ stateOf(id: string): JsonObject | undefined { return this.#states?.value(id); } /** The run state documents, for `ctx.state.change`; `undefined` without a state definition. */ - get states(): DocumentSlot | undefined { + get states(): RunStates | undefined { return this.#states; } + /** Whether the job persists its runs. */ + get persisted(): boolean { + return this.#persisted !== undefined; + } + /** - * The object ends: running runs' signals fire, every waiter rejects with `error`, blocked runs are dropped - * (reported); nothing finishes. The records stay readable until `drop`. + * The object ends: running runs' signals fire, every waiter rejects with `error`; blocked in-memory runs are + * dropped (reported), persisted ones keep their records; nothing finishes. The records stay readable until `drop`. */ end(error: ChordError): void { if (this.#ended !== undefined) return; @@ -238,9 +464,11 @@ export class JobRecords { this.#job = undefined; this.#waiting.clear(); for (const run of this.#runs.values()) { - if (run.phase === "blocked") this.report({ kind: "job_dropped", id: run.id }); - run.abort.abort(error); + if (run.phase === "blocked" && this.#persisted === undefined) this.report({ kind: "job_dropped", id: run.id }); + const { invocation } = run; + invocation?.abort.abort(error); run.reject(error); + if (invocation?.state !== "running") invocation?.ended(); } if (this.#active > 0) this.#owner.countRuns(-this.#active); } @@ -250,26 +478,87 @@ export class JobRecords { this.#runs.clear(); this.#keys.clear(); this.#states?.drop(); + this.#persisted?.drop(); this.running.end(); } - /** Reports a run's failure or drop to the host. */ + /** Reports a run's failure, drop or interruption to the host. */ report( error: | { readonly kind: "job_failed" | "job_finish_failed"; readonly id: string; readonly error: unknown } - | { readonly kind: "job_dropped"; readonly id: string }, + | { readonly kind: "job_dropped"; readonly id: string } + | { readonly kind: "job_interrupted"; readonly id: string; readonly restarts: number }, ): void { this.#owner.host.report({ ...error, object: this.#owner.label, job: this.definition.name }); } - /** Records the outcome of `run`, then has it finished on the live attempt. */ + /** The store writes of `tx` (Chord's own documents of a persisted job). */ + #store(tx: Transaction): StagedWrite[] { + return tx.stageStore(this.#owner.store!.store); + } + + /** + * The run a loaded record stands for and the change its activation writes (D6): a record of another version or + * fingerprint blocked as stored (its state document only found, never decoded: slice 7 maps it); otherwise its state + * document read and decoded (migrated), pinned while it is unsettled, a crash counted, at the limit its outcome, + * marked resuming unless blocked or finishing. + */ + async #restored(loaded: LoadedRecord, context: Context): Promise { + const { id, record } = loaded; + const { definition } = this; + const run = new JobRun(id, record.version, record.key, record.args, record.phase, record.trace); + run.restarts = record.restarts; + run.finishFailures = record.finishFailures; + run.abortMarked = record.abort === true; + const states = this.#states; + if (record.version !== definition.version || record.fingerprint !== definition.fingerprint) { + run.phase = "blocked"; + run.blocked = "version"; + // What is stored of its state (also when this version declares none), so an abort drops it with the record. + await this.#persisted!.findBlockedState(id, context); + return { run, change: undefined }; + } + const document = await states?.persistence.read(id, context); + if (document !== undefined) { + const { value, stored } = states!.persistence.decode(document, states!.definition.current, context); + states!.create(id, value, stored); + } + if (record.outcome !== undefined) run.outcome = outcomeOf(record.outcome, id, record.args, this.stateOf(id)); + run.blocked = record.blocked; + switch (record.phase) { + case "running": { + const restarts = record.restarts + 1; + run.resuming = true; + if (restarts < RESTART_LIMIT) { + return { run, change: { kind: "update", run, fields: { phase: "pending", restarts } } }; + } + run.restarts = restarts; + const outcome = this.#outcome(run, { kind: "failed", error: { kind: "restart_limit", restarts } }); + return { run, change: { kind: "outcome", run, outcome } }; + } + case "stopped": + run.resuming = true; + return { run, change: { kind: "update", run, fields: { phase: "pending" } } }; + case "pending": + run.resuming = true; + return { run, change: undefined }; + default: + return { run, change: undefined }; + } + } + + /** Records the outcome of `run` (with what its state holds, D8), then has it finished on the live attempt. */ async #recordOutcome(run: JobRun, ended: Settled): Promise { const { documents, context } = this.#owner; try { await documents.tx( (tx) => { - const outcome = { id: run.id, args: run.args, state: this.stateOf(run.id), ...ended }; - this.staging(tx).changes.push({ kind: "outcome", run, outcome }); + const outcome = this.#outcome(run, ended); + this.change(tx, { kind: "outcome", run, outcome }); + // What its state holds is stored before the outcome is recorded (D8). + const adopt = + this.#persisted === undefined ? undefined : this.#states?.stageHeld(run.id, this.#store(tx)); + if (adopt !== undefined) this.staging(tx).adopts.push(adopt); }, documents.own, context, @@ -277,10 +566,18 @@ export class JobRecords { } catch { // The object ended: nothing finishes. return; + } finally { + // A run ending without an invocation (marked aborted, abandoned, cut off in memory) has none to wait for. + run.invocation?.ended(); } this.#need(run); } + /** `run`'s outcome for `ended`, with its args and its state as it is now. */ + #outcome(run: JobRun, ended: Settled): AnyOutcome { + return { id: run.id, args: run.args, state: this.stateOf(run.id), ...ended }; + } + /** `run` needs a live attempt (to be invoked or finished): now, or once one attaches. */ #need(run: JobRun): void { if (this.#ended !== undefined || this.#runs.get(run.id) !== run) return; @@ -289,30 +586,53 @@ export class JobRecords { else this.#dispatch(run, job); } - /** Invokes or finishes `run` on `job`, as its record stands. */ + /** + * Invokes or finishes `run` on `job`, as its record stands: a finishing run is finished; a run not invoked yet is + * invoked, unless it is marked aborted (it ends `aborted` without `run`) or resumes abandoned (`failed { aborted, + * reason: "restart" }`). + */ #dispatch(run: JobRun, job: Job): void { if (this.#runs.get(run.id) !== run) return; - if (run.phase === "finishing") job.finish(run); - else if (run.phase === "running" && run.invocation === "pending") job.invoke(run); + if (run.phase === "finishing") { + job.finish(run); + return; + } + if (run.phase === "blocked" || run.invoked) return; + if (run.abortMarked) { + void this.#recordOutcome(run, ABORTED); + return; + } + if (run.resuming && this.definition.persist?.resume === "abandon") { + void this.#recordOutcome(run, RESTART); + return; + } + run.resuming = false; + job.invoke(run); } /** - * Adopts one record change, inside the commit step: the maps, the active count, the state document, `running`. A - * new run is counted for its object here, before the commit's publication (a link closing while it sends may - * release the object's last reference) and before any run of the same commit settles in `#committed`. + * Adopts one record change, inside the commit step: the maps, the active count, `running`. A new run is counted for + * its object here, before the commit's publication (a link closing while it sends may release the object's last + * reference) and before any run of the same commit settles in `#committed`. */ - #adopt(change: RecordChange): void { + #adopt(change: Adopted): void { const { run } = change; switch (change.kind) { case "create": + case "restore": this.#runs.set(run.id, run); if (run.key !== undefined) this.#keys.set(run.key, run); - this.#active++; - this.#owner.countRuns(1); - if (change.state !== undefined) this.#states?.create(run.id, change.state); - run.entry = run.entryFor(change.state); + if (run.phase !== "blocked") { + this.#active++; + this.#owner.countRuns(1); + } + run.entry = run.entryFor(this.stateOf(run.id)); this.running.add(run.entry); return; + case "update": + Object.assign(run, change.fields); + this.#replaceEntry(run); + return; case "outcome": run.outcome = change.outcome; run.phase = "finishing"; @@ -320,6 +640,7 @@ export class JobRecords { return; case "block": run.phase = "blocked"; + run.blocked = change.reason; this.#active--; this.#replaceEntry(run); return; @@ -329,7 +650,6 @@ export class JobRecords { // The finishing run's key may name its successor already, started in the same commit. if (run.key !== undefined && this.#keys.get(run.key) === run) this.#keys.delete(run.key); if (change.kind === "delete") this.#active--; - this.#states?.retire(run.id); this.running.remove(run.entry!); run.entry = undefined; return; @@ -337,13 +657,15 @@ export class JobRecords { } /** The actions after a record change committed. */ - #committed(change: RecordChange): void { + #committed(change: Adopted): void { const { run } = change; switch (change.kind) { case "create": // After the commit, outside the batch, once the transaction settled; on the live attempt, or the next. queueMicrotask(() => this.#need(run)); return; + case "restore": + case "update": case "outcome": return; case "delete": @@ -362,8 +684,10 @@ export class JobRecords { /** A run's state document changed by `ops`, inside the commit step: its entry is replaced. */ #stateChanged(id: string, ops: readonly Op[]): void { - const run = this.#runs.get(id)!; - const before = run.entry!; + const run = this.#runs.get(id); + // Its record may not be adopted yet (a start creates both in one commit; its entry takes the state then). + if (run?.entry === undefined) return; + const before = run.entry; run.entry = run.entryFor(this.stateOf(id)); this.running.stateChanged(before, run.entry, ops); } @@ -384,63 +708,21 @@ export class JobRecords { } } -/** - * A job's state documents, one per run by its id: Chord's own keyed slot, apart from the object's `self.state` slots. - * Created with the run and retired with its record; never drafted while absent, so a retired one is never recreated. - * Each commit tells the job, whose `running` carries the state. - */ -class RunStates implements DocumentSlot { - readonly owner: Documents; - /** In memory only (persisted jobs are slice 6b's). */ - readonly persisted = undefined; - readonly #definition: StateDefinition; - readonly #trackers = new Map>(); - readonly #changed: (id: string, ops: readonly Op[]) => void; +/** Runs every adoption of `adopts`. */ +function runAll(adopts: readonly (() => void)[]): void { + for (const adopt of adopts) adopt(); +} - constructor( - definition: StateDefinition, - owner: Documents, - changed: (id: string, ops: readonly Op[]) => void, - ) { - this.#definition = definition; - this.owner = owner; - this.#changed = changed; - } - - /** The value of run `id`'s document; `undefined` once retired. */ - value(id: string): JsonObject | undefined { - return this.#trackers.get(id)?.value; - } - - current(key: string): Tracker | undefined { - return this.#trackers.get(key); - } - - initial(_context: Context): JsonObject { - throw new ChordError({ - code: "released", - reason: "settled", - message: `${this.#definition.name}: the run's document is retired`, - }); - } - - adopt(key: string, tracker: Tracker, prepared: Prepared): void { - tracker.adopt(prepared); - this.#changed(key, prepared.ops); - } - - /** Retires run `key`'s document (its record's deletion). */ - retire(key: string): void { - this.#trackers.delete(key); - } - - /** Creates run `id`'s document with `value` (its start's commit). */ - create(id: string, value: JsonObject): void { - this.#trackers.set(id, track(value)); - } - - /** Drops every document (the object ended). */ - drop(): void { - this.#trackers.clear(); +/** The record fields a change writes, its phase transitions included. */ +function recordFields(change: Exclude): StoredFields { + switch (change.kind) { + case "create": + return {}; + case "update": + return change.fields; + case "outcome": + return { phase: "finishing", outcome: change.outcome }; + case "block": + return { phase: "blocked", blocked: change.reason }; } } diff --git a/packages/chord/src/next/jobs/run-states.ts b/packages/chord/src/next/jobs/run-states.ts new file mode 100644 index 000000000..f8305c44c --- /dev/null +++ b/packages/chord/src/next/jobs/run-states.ts @@ -0,0 +1,243 @@ +// A job's run state documents (§15, slice 6b §4.1, §4.3): one per run by its id, Chord's own slot apart from the +// object's `self.state` slots. In memory, or persisted in the object's store (namespace `chord.job-states`), always in +// memory while the run is unsettled (`ctx.state.value` reads it synchronously). A persisted run's deferred changes +// (D8) are adopted and published at once and held, stored before the next change's own, at the outcome, at close or +// when their deadline passes. + +import type { Context } from "../../context/index.ts"; +import { type Op, PendingOps, type Prepared, type Tracker, track } from "../../delta/index.ts"; +import { ChordError } from "../data/index.ts"; +import type { CommitPersistence, JsonObject, StateDefinition } from "../documents/index.ts"; +import { + type DocumentChange, + type DocumentSlot, + type Documents, + type Held, + type PersistedSlot, + type Persistence, + SlotDefinition, + type Stored, + type StoredChange, + storedAfter, +} from "../documents/internal.ts"; +import { changeDocument, createDocument } from "../store/engine.ts"; +import type { StagedWrite } from "../store/step.ts"; + +/** + * A held state's ops, folded as they come (`PendingOps`: repeated writes of one leaf keep one op; past its bound only + * "store a base" stays), whether a held change asked for a base, and when it must be stored at the latest. + */ +type HeldState = { readonly pending: PendingOps; base: boolean; deadline: number }; + +/** What the run states tell their job: a document changed (`running` follows), a held change's deadline. */ +export type RunStatesOwner = { + readonly documents: Documents; + /** Run `id`'s document changed by `ops`, inside the commit step. */ + changed(id: string, ops: readonly Op[]): void; + /** A held change must be stored by `deadline` (`Date.now()` time). */ + held(deadline: number): void; +}; + +/** How long a deferred change is held at most by default (Durable's `progress.persistWithinMs`). */ +const DEFAULT_WITHIN_MS = 1_000; + +/** A run's state documents. Created with the run, retired with its record; never drafted while absent. */ +export class RunStates implements DocumentSlot, PersistedSlot { + readonly owner: Documents; + /** The store side when the job persists; `undefined` in memory. */ + readonly persisted: PersistedSlot | undefined; + readonly definition: SlotDefinition; + readonly #persistence: Persistence | undefined; + readonly #states: RunStatesOwner; + readonly #trackers = new Map>(); + readonly #stored = new Map(); + readonly #held = new Map(); + + constructor( + definition: StateDefinition, + states: RunStatesOwner, + persistence: Persistence | undefined, + ) { + this.definition = new SlotDefinition(definition); + this.owner = states.documents; + this.#states = states; + this.#persistence = persistence; + this.persisted = persistence === undefined ? undefined : this; + } + + get persistence(): Persistence { + // Only reached through `persisted`, which a persisted job's states alone have. + return this.#persistence!; + } + + /** The value of run `id`'s document; `undefined` once retired. */ + value(id: string): JsonObject | undefined { + return this.#trackers.get(id)?.value; + } + + current(key: string): Tracker | undefined { + return this.#trackers.get(key); + } + + initial(_context: Context): JsonObject { + throw new ChordError({ + code: "released", + reason: "settled", + message: `${this.definition.current.name}: the run's document is retired`, + }); + } + + /** Pinned in memory while the run is unsettled: never cold. */ + cold(_key: string): boolean { + return false; + } + + stored(key: string): Stored | undefined { + return this.#stored.get(key); + } + + async load(_key: string, _context: Context): Promise {} + + /** + * A change of run `key`'s document: a deferred one of a stored document is held (nothing written); an immediate + * one stores what is held with its own ops, as one delta on the stored revision or a base. + */ + stage( + key: string, + change: DocumentChange, + was: Stored | undefined, + writes: StagedWrite[], + persistence: CommitPersistence, + ): StoredChange { + if (persistence.kind === "deferred" && was !== undefined && !change.retires && change.prepared !== undefined) { + return { kind: "held", stored: was, withinMs: persistence.withinMs ?? DEFAULT_WITHIN_MS }; + } + const held = this.#held.get(key); + return this.persistence.stage(key, this.definition.current, was, change, writes, held && heldOps(held)); + } + + adopt( + key: string, + tracker: Tracker, + prepared: Prepared, + change: StoredChange | undefined, + ): void { + tracker.adopt(prepared); + if (change?.kind === "held") this.#hold(key, prepared, change); + else if (change !== undefined) this.#settle(key, storedAfter(change)); + this.#states.changed(key, prepared.ops); + } + + /** Retires run `key`'s document (its record's deletion): what is held goes with it. */ + retire(key: string): void { + this.#trackers.delete(key); + this.#stored.delete(key); + this.#held.delete(key); + } + + /** Run `id`'s document with `value`, as its start created it or a load found it (`stored`: persisted). */ + create(id: string, value: JsonObject, stored: Stored | undefined): void { + this.#trackers.set(id, track(value)); + if (stored !== undefined) this.#stored.set(id, stored); + } + + /** + * Stages the creation of run `id`'s document with `value` into `writes` (its start); returns the adoption, which + * creates it in memory with what is stored. + */ + stageCreate(id: string, value: JsonObject, writes: () => StagedWrite[]): () => void { + const persistence = this.#persistence; + if (persistence === undefined) return () => this.create(id, value, undefined); + const { version, fingerprint } = this.definition.current; + const created = { id: 0 }; + const base = { version, fingerprint, value }; + writes().push( + createDocument(persistence.at(id), base, JSON.stringify(value), (stored) => { + created.id = stored; + }), + ); + return () => this.create(id, value, { id: created.id, revision: 1, version, deltasSinceBase: 0 }); + } + + /** When run `id`'s held changes must be stored at the latest (`Infinity`: none held, or none with a deadline). */ + deadline(id: string): number { + return this.#held.get(id)?.deadline ?? Number.POSITIVE_INFINITY; + } + + /** The runs whose documents hold changes not stored yet. */ + get heldIds(): readonly string[] { + return [...this.#held.keys()]; + } + + /** + * Stages what run `id`'s document holds into `writes` (a flush: its deadline, the outcome, the close writer), as one + * delta on its stored revision or a base; returns the adoption, `undefined` when nothing is held. + */ + stageHeld(id: string, writes: StagedWrite[]): (() => void) | undefined { + const held = this.#held.get(id); + const stored = this.#stored.get(id); + const tracker = this.#trackers.get(id); + if (held === undefined || stored === undefined || tracker === undefined) return undefined; + const { version, fingerprint } = this.definition.current; + const value = tracker.value; + const { ops, base: asked } = heldOps(held); + const base = asked || stored.version !== version; + const content = base + ? ({ kind: "base", version, fingerprint, value } as const) + : ({ kind: "delta", version, ops } as const); + writes.push(changeDocument(stored.id, stored.revision, content, JSON.stringify(base ? value : ops))); + const deltasSinceBase = base ? 0 : stored.deltasSinceBase + 1; + const after: Stored = { id: stored.id, revision: stored.revision + 1, version, deltasSinceBase }; + return () => { + if (this.#held.get(id) === held) this.#settle(id, after); + }; + } + + /** Drops every document (the object ended): what is held is lost, as a crash loses it. */ + drop(): void { + this.#trackers.clear(); + this.#stored.clear(); + this.#held.clear(); + } + + /** What is stored of run `key`'s document after a commit stored it: nothing is held any more. */ + #settle(key: string, stored: Stored | undefined): void { + if (stored === undefined) this.#stored.delete(key); + else this.#stored.set(key, stored); + this.#held.delete(key); + } + + /** A deferred change adopted: its ops held (a base when `checkpointWhen` asks), its deadline told. */ + #hold(key: string, prepared: Prepared, change: Extract): void { + const held = this.#held.get(key) ?? { + pending: new PendingOps(), + base: false, + deadline: Number.POSITIVE_INFINITY, + }; + const deltasSinceBase = change.stored.deltasSinceBase + (held.pending.size > 0 ? 1 : 0); + if (this.definition.current.checkpointWhen(prepared.value, prepared.ops, { deltasSinceBase })) held.base = true; + // A base stores the whole value: no op needs keeping for it. + if (held.base) held.pending.take(); + else held.pending.push(prepared.ops); + held.deadline = Math.min(held.deadline, Date.now() + change.withinMs); + this.#held.set(key, held); + if (Number.isFinite(held.deadline)) this.#states.held(held.deadline); + } +} + +/** + * What `held` stores, its queue left as it was (a commit may still be refused): its folded ops, or a base once a held + * change asked for one or the queue overflowed. Taking and pushing back one batch keeps it by reference: no copy. + */ +function heldOps(held: HeldState): Held { + if (held.base) return NO_HELD_OPS_BASE; + const taken = held.pending.take(); + if (taken.kind === "replace") { + held.base = true; + return NO_HELD_OPS_BASE; + } + held.pending.push(taken.ops); + return { ops: taken.ops, base: false }; +} + +const NO_HELD_OPS_BASE: Held = { ops: [], base: true }; diff --git a/packages/chord/src/next/jobs/run.ts b/packages/chord/src/next/jobs/run.ts index 688e0efa1..cff213177 100644 --- a/packages/chord/src/next/jobs/run.ts +++ b/packages/chord/src/next/jobs/run.ts @@ -1,5 +1,7 @@ -// One run of a job (§15): its record (id, version, key, args, phase, outcome), its abort signal, the fence of its -// invocation (its seal token), its `running` entry and the `Run` handle its callers wait on. +// One run of a job (§15, slice 6b §4.1): its record (id, version, key, args, phase, restarts, finish failures, abort +// mark, outcome), its current invocation (the fence of its seal token, its abort signal, the end an abort joins: one +// per invocation, as a persisted run is invoked again after a seal, R3), its `running` entry and the `Run` handle its +// callers wait on. import type { Context } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; @@ -23,13 +25,15 @@ export interface Run { export type AnyOutcome = Outcome; /** - * Where a run's record stands. `running` from its start's commit (its `run` starts after it), `finishing` once its - * outcome is recorded, `blocked` after five failed finishes. + * Where a run's record stands. A persisted run is `pending` from its start's commit until its reservation commit + * (`running`, then its `run` starts), `stopped` once a clean stop cut it off; an in-memory run is `running` from its + * start's commit. `finishing` once its outcome is recorded, `blocked` after five failed finishes or for another + * version. */ -type Phase = "running" | "finishing" | "blocked"; +export type Phase = "pending" | "running" | "stopped" | "finishing" | "blocked"; -/** Where a run's invocation stands: `ctx.state.change` is admitted only while it runs (the invocation fence). */ -type Invocation = "pending" | "running" | "settled"; +/** Why a run is blocked: its `finish` failed five times in a row, or it was stored by another job version (slice 7). */ +export type BlockedReason = "finish_failed" | "version"; /** How a run settled for its waiters. */ type Answer = @@ -45,13 +49,21 @@ export class JobRun { readonly version: number; readonly key: string | undefined; readonly args: JsonValue; - /** `run`'s abort signal: `abort(id)`, the object's end and `host.close` fire it; an early abort pre-marks it. */ - readonly abort = new AbortController(); readonly handle: Run; - /** The seal token of its invocation: sealed when the invocation is cut off by its attempt's seal (`ctx.state.change`). */ - readonly token = new SealToken(); - phase: Phase = "running"; - invocation: Invocation = "pending"; + /** Its current invocation, from its reservation (or its start in memory) on; a later one replaces it. */ + invocation: Invocation | undefined; + phase: Phase; + /** A persisted run's crashes in a row while running (D6); reset by a state change. */ + restarts = 0; + /** Failed finishes in a row (stored for a persisted run, so they count across a crash). */ + finishFailures = 0; + /** Set by `abort(id)` (stored for a persisted run, D7): a run not invoked yet ends `aborted` without `run`. */ + abortMarked = false; + /** A persisted run cut off by a crash or a clean stop, to be resumed per its job's `resume` (no new start). */ + resuming = false; + blocked: BlockedReason | undefined; + /** The start's trace carrier, when the adapter propagates (stored, slice 9 uses it). */ + readonly trace: { readonly [key: string]: string } | undefined; /** What it ended with, once recorded (finishing and blocked). */ outcome: AnyOutcome | undefined; /** Its entry in `running`, while its record exists; replaced when its record or state document changes. */ @@ -59,29 +71,62 @@ export class JobRun { /** Its waiters: apart from the run, so a handle kept by a caller keeps only how the run settled. */ readonly #waiters = new Waiters(); - constructor(id: string, version: number, key: string | undefined, args: JsonValue) { + constructor( + id: string, + version: number, + key: string | undefined, + args: JsonValue, + phase: Phase, + trace?: { readonly [key: string]: string }, + ) { this.id = id; this.version = version; this.key = key; this.args = args; + this.phase = phase; + this.trace = trace; const waiters = this.#waiters; this.handle = { id, outcome: (context) => waiters.wait(context) }; } - /** Its `running` entry for its current record and `state` (the job's state document, `undefined` without one). */ + /** + * Its `running` entry for its current record and `state` (the job's state document, `undefined` without one): a + * pending or stopped run shows as `running` (it runs, or will run again). + */ entryFor(state: JsonValue | undefined): JsonValue { const { id, version, key, args, phase } = this; - if (phase === "blocked") return { id, version, phase, reason: "finish_failed" }; + if (phase === "blocked") return { id, version, phase, reason: this.blocked! }; return { id, version, ...(key === undefined ? {} : { key }), args, ...(state === undefined ? {} : { state }), - phase, + phase: phase === "finishing" ? "finishing" : "running", }; } + /** A new invocation, the current one from now on (a run marked aborted is never invoked: it ends without one). */ + begin(): Invocation { + const invocation = new Invocation(); + this.invocation = invocation; + return invocation; + } + + /** Whether its current invocation runs (or is reserved): it is not invoked again meanwhile. */ + get invoked(): boolean { + return this.invocation?.state === "running"; + } + + /** + * Resolves once its current invocation ended, at once when none runs (`abort` joins it; one a seal cut off is + * fenced and is not waited for); `context` ends only the wait. + */ + over(context: Context): Promise { + const invocation = this.invocation; + return invocation?.state === "running" ? invocation.over(context) : Promise.resolve(); + } + /** Its waiters get the outcome. */ resolve(outcome: AnyOutcome): void { this.#waiters.settle({ kind: "done", outcome }); @@ -93,6 +138,35 @@ export class JobRun { } } +/** + * One invocation of a run's `run`: its fence (`ctx.state.change` admitted while it runs, under its own seal token, + * which its attempt's seal fences), its abort signal and the waits for its end. Captured by its ctx and its settling, + * so a replaced invocation never acts for its successor. + */ +export class Invocation { + readonly token = new SealToken(); + /** `run`'s abort signal: `abort(id)`, the attempt's seal, the object's end and `host.close` fire it. */ + readonly abort = new AbortController(); + /** `running` from its reservation until `run` settled or a seal cut it off. */ + state: "running" | "settled" = "running"; + readonly #joins = new Joins(); + + /** The fence closes: a change made later is refused. */ + settle(): void { + this.state = "settled"; + } + + /** Nothing of it runs here any more (its `run` settled, or the object ended): every join resolves. */ + ended(): void { + this.#joins.end(); + } + + /** Resolves once it ended; `context` ends only the wait. */ + over(context: Context): Promise { + return this.#joins.wait(context); + } +} + /** A run's waits for its outcome: each removed when its context aborts; late ones answered at once. */ class Waiters { /** How the run settled, once it did. */ @@ -135,3 +209,36 @@ class Waiters { function answer(settled: Answer): Promise { return settled.kind === "done" ? Promise.resolve(settled.outcome) : Promise.reject(settled.error); } + +/** The waits for a run's invocation to be over: each removed when its context aborts, so a cancelled one keeps nothing. */ +class Joins { + #over = false; + readonly #waiting = new Set<() => void>(); + + /** The invocation is over (once): every wait resolves. */ + end(): void { + if (this.#over) return; + this.#over = true; + for (const resolve of this.#waiting) resolve(); + this.#waiting.clear(); + } + + /** A wait: at once when over; otherwise until it is or `context` aborts (rejecting with its reason). */ + wait(context: Context): Promise { + if (this.#over) return Promise.resolve(); + const signal = context.abortSignal; + if (signal?.aborted) return Promise.reject(signal.reason); + const { promise, resolve, reject } = Promise.withResolvers(); + const abort = (): void => { + this.#waiting.delete(done); + reject(signal?.reason); + }; + const done = (): void => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + this.#waiting.add(done); + signal?.addEventListener("abort", abort, { once: true }); + return promise; + } +} diff --git a/packages/chord/src/next/jobs/running.ts b/packages/chord/src/next/jobs/running.ts index 3f24ccd24..098cb2804 100644 --- a/packages/chord/src/next/jobs/running.ts +++ b/packages/chord/src/next/jobs/running.ts @@ -7,6 +7,7 @@ import type { NonEmptyPath, Op, Path } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; import { ChordError } from "../data/index.ts"; import { type Batch, Source } from "../state/internal.ts"; +import type { BlockedReason } from "./run.ts"; /** * One entry of `running`: a running or finishing run with its args (and state, when the job has a state document), or @@ -20,7 +21,7 @@ export type RunningEntry = readonly args: A; readonly phase: "running" | "finishing"; } & ([S] extends [undefined] ? unknown : { readonly state: S })) - | { readonly id: string; readonly version: number; readonly phase: "blocked"; readonly reason: "finish_failed" }; + | { readonly id: string; readonly version: number; readonly phase: "blocked"; readonly reason: BlockedReason }; const EMPTY: readonly JsonValue[] = []; diff --git a/packages/chord/src/next/jobs/stored.ts b/packages/chord/src/next/jobs/stored.ts new file mode 100644 index 000000000..e2f14405f --- /dev/null +++ b/packages/chord/src/next/jobs/stored.ts @@ -0,0 +1,106 @@ +// A persisted run's record (slice 6b §4.1, D5): the document each unsettled run of a persisted job keeps in its +// object's store (namespace `chord.jobs`, kind the job name, key the run id), and the conversion of an outcome to and +// from it. Errors are stored in the wire error format, so a `PluginError`'s code, message and details survive. + +import type { JsonValue } from "../../json.ts"; +import type { JsonObject } from "../documents/index.ts"; +import { errorFromWire, wireError } from "../link/internal.ts"; +import type { WireError } from "../link/message.ts"; +import type { JobError } from "./definition.ts"; +import type { AnyOutcome, BlockedReason, JobRun, Phase } from "./run.ts"; + +/** How a run failed, as stored: a thrown error in the wire format, an abort, the restart limit. */ +export type StoredJobError = + | { readonly kind: "threw"; readonly error: WireError } + | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } + | { readonly kind: "restart_limit"; readonly restarts: number }; + +/** A run's outcome as stored: its result, or how it failed. */ +export type StoredOutcome = + | { readonly kind: "done"; readonly result: JsonValue } + | { readonly kind: "failed"; readonly error: StoredJobError }; + +/** The record document of one unsettled persisted run. */ +export type StoredRun = { + readonly version: number; + readonly fingerprint: string; + readonly args: JsonValue; + readonly key?: string; + readonly trace?: { readonly [key: string]: string }; + readonly phase: Phase; + readonly restarts: number; + readonly finishFailures: number; + readonly abort?: true; + readonly blocked?: BlockedReason; + readonly outcome?: StoredOutcome; +}; + +/** + * The record documents' version and fingerprint (Chord's own documents: no definition, no `migrate`; a new record + * shape raises the version). + */ +export const RUN_RECORD = { version: 1, fingerprint: "chord.job-run.1" } as const; + +/** What a change sets on a run's record; the rest is the run's own. */ +export type StoredFields = { + readonly phase?: Phase; + readonly restarts?: number; + readonly finishFailures?: number; + readonly abortMarked?: true; + readonly blocked?: BlockedReason; + readonly outcome?: AnyOutcome; +}; + +/** `run`'s record as stored with `fields` changed, with the job's `fingerprint`. */ +export function storedRun(run: JobRun, fingerprint: string, fields: StoredFields = {}): StoredRun { + const { version, args, key, trace } = run; + const { phase, restarts, finishFailures, abortMarked, blocked, outcome } = { ...pick(run), ...fields }; + return { + version, + fingerprint, + args, + ...(key === undefined ? {} : { key }), + ...(trace === undefined ? {} : { trace }), + phase, + restarts, + finishFailures, + ...(abortMarked ? { abort: true as const } : {}), + ...(blocked === undefined ? {} : { blocked }), + ...(outcome === undefined ? {} : { outcome: storedOutcome(outcome) }), + }; +} + +/** The fields of `run` a record stores that change. */ +function pick(run: JobRun): Required> & StoredFields { + const { phase, restarts, finishFailures, blocked, outcome } = run; + return { phase, restarts, finishFailures, ...(run.abortMarked ? { abortMarked: true } : {}), blocked, outcome }; +} + +/** An outcome as stored: a result as it is (JSON: persisted jobs refuse `bytes()`), an error in the wire format. */ +function storedOutcome(outcome: AnyOutcome): StoredOutcome { + if (outcome.kind === "done") return { kind: "done", result: outcome.result as JsonValue }; + return { kind: "failed", error: storedError(outcome.error) }; +} + +function storedError(error: JobError): StoredJobError { + return error.kind === "threw" ? { kind: "threw", error: wireError(error.error) } : error; +} + +/** The outcome a stored one stands for, rebuilt as the in-memory one (`id`, `args` and `state` from the run). */ +export function outcomeOf( + stored: StoredOutcome, + id: string, + args: JsonValue, + state: JsonValue | undefined, +): AnyOutcome { + if (stored.kind === "done") return { id, args, state, kind: "done", result: stored.result }; + const { error } = stored; + const rebuilt: JobError = error.kind === "threw" ? { kind: "threw", error: errorFromWire(error.error) } : error; + return { id, args, state, kind: "failed", error: rebuilt }; +} + +/** A stored record as Chord wrote it (the record definition's value, a JSON object). */ +export function recordOf(value: JsonObject): StoredRun { + // Chord's own document: `storedRun` wrote it. + return value as unknown as StoredRun; +} diff --git a/packages/chord/src/next/remote/hold.ts b/packages/chord/src/next/remote/hold.ts index a87c4aaa8..232d2122a 100644 --- a/packages/chord/src/next/remote/hold.ts +++ b/packages/chord/src/next/remote/hold.ts @@ -372,6 +372,7 @@ export class Hold implements Holder { }, // The member's source ended with its object, which ends this hold. end: () => {}, + endAfterDelivery: () => {}, }, encoder: encoder(), pending: new PendingOps(), diff --git a/packages/chord/src/next/state/batch.ts b/packages/chord/src/next/state/batch.ts index d58c5c347..66dbd5d5a 100644 --- a/packages/chord/src/next/state/batch.ts +++ b/packages/chord/src/next/state/batch.ts @@ -27,6 +27,8 @@ export class Batch { #sending = false; /** Changes `within` deferred while the flushes sent, for the next commit. */ #later: (() => void)[] = []; + /** Changes `soon` collected for the commit on the next microtask, while one is queued. */ + #soon: (() => void)[] | undefined; constructor(report: (error: BatchError) => void, metrics: TelemetryMetrics | undefined) { this.#report = report; @@ -34,6 +36,11 @@ export class Batch { this.#metrics = metrics; } + /** Whether a commit's adoption, a delivery or a derive compute runs now: a change made now lands in this batch. */ + get running(): boolean { + return this.#running; + } + /** Throws `invalid/reentrant` inside the batch: `operation` may not run during adoptions or derive computes. */ assertOutside(operation: string): void { if (this.#running) throw invalid("reentrant", `${operation} inside a commit or a derive compute`); @@ -101,6 +108,26 @@ export class Batch { else this.commit(adopt, context); } + /** + * Runs `adopt` in one commit on the next microtask, together with every other `soon` until then (changes arriving + * outside any commit, as a followed runtime's calls off its commit step: one batch for all of them). `context` is + * the first one's. + */ + soon(adopt: () => void, context: Context): void { + if (this.#soon !== undefined) { + this.#soon.push(adopt); + return; + } + const soon = [adopt]; + this.#soon = soon; + queueMicrotask(() => { + this.#soon = undefined; + this.commit(() => { + for (const each of soon) each(); + }, context); + }); + } + /** Lists a watched source the current commit changed. */ changed(source: Source): void { this.#changed.push(source); diff --git a/packages/chord/src/next/state/watchable.ts b/packages/chord/src/next/state/watchable.ts index 8bdbac622..a7fd841ae 100644 --- a/packages/chord/src/next/state/watchable.ts +++ b/packages/chord/src/next/state/watchable.ts @@ -20,6 +20,11 @@ export interface Subscriber { advance(call: WatchCall, context: Context): void; /** The source ended for good with `end`; nothing follows, and the subscriber is dropped. */ end(end: WatchEnd): void; + /** + * The source ended for good after its last call (a followed runtime ended): a subscriber delivering later (a watch) + * ends `ended` once it delivered what it was given, others end now. + */ + endAfterDelivery(): void; } /** @@ -147,6 +152,18 @@ export abstract class Source implements Watchable { this.forward({ kind: "failed", error: handedOut(error) }, context); } + /** + * Ends every watch and subscriber for good once what they were given is delivered (a followed runtime ended after a + * last call, Durable's `endAfterDelivery`): `ended`; nothing follows. + */ + protected endWatchesAfterDelivery(): void { + for (const watch of [...this.#watches]) watch.endAfterDelivery(); + if (this.#subscribers.size === 0) return; + const subscribers = [...this.#subscribers]; + this.#subscribers.clear(); + for (const subscriber of subscribers) subscriber.endAfterDelivery(); + } + /** Ends every watch and subscriber for good (its owner ended): `ended`; nothing follows. */ protected endWatches(): void { // Ending detaches, which deletes the current entry (safe while iterating) and runs no listener. diff --git a/packages/chord/src/next/store/README.md b/packages/chord/src/next/store/README.md index 933fb6df6..b1fcb2975 100644 --- a/packages/chord/src/next/store/README.md +++ b/packages/chord/src/next/store/README.md @@ -52,6 +52,18 @@ const copy = await importStore(sqliteBackend(db), decodeJsonl(bytes), ctx); checks every part (a trust boundary) and applies the archive in one backend transaction; JSONL via `encodeJsonl` and `decodeJsonl`. +## For adapters (`../index.ts`, marked internal) + +Durable's `StoreStorage` keeps its records in the store: `StoreOwnerTables` declares and prepares an owner's tables +(claimed, `":"`), `OwnerTable` encodes rows at staging, writes them in a step (`put`, with a step-stamped +column) and reads them through queries compiled once (`prepare`, `prepareFirst`, `prepareGet`, `reader`); the documents +engine's writes (`insertDocument`, `writeContent`, `retireRecord`) and reads at a point (`findDocument`, `loadContent`, +`listDocumentEntries`, …); `store.commitInBatch` adopts in the attached host's batch; `store.mintId()` and `store.ids` +are the one allocator (D3); `store.claims` holds what can collide; `store.readTransaction` reads several queries at one +state. A step's own error is a refusal (the commit rejects with it, the store goes on); `refusalOf` classifies it for +Chord's owners. `applyBatch(store, commits, ctx)` replays outbox commits (the sync layer's entry point): refused at its +call and at its turn while an owner holds a claim or once the store diverged, and no claim is taken while it applies. + ## For Chord's layers `ChordStore` (`store.ts`: `commit(staged, fence, ctx)`, `read(fn, ctx)` guarded as Durable's `GuardedStorage`, diff --git a/packages/chord/src/next/store/adapter.ts b/packages/chord/src/next/store/adapter.ts new file mode 100644 index 000000000..6e5eec4b6 --- /dev/null +++ b/packages/chord/src/next/store/adapter.ts @@ -0,0 +1,152 @@ +// The store for adapters (slice 6b §2.1, D1, D2): what an owner outside Chord's objects (Durable's `StoreStorage`) uses +// to keep its data in the store as declared tables and engine documents: its tables declared and prepared under its +// owner name (claimed, the path `self.table` takes), rows encoded at staging and written in a step with what only the +// step knows, and its reads. Internal for adapters; apps use `openStore` and objects. + +import type { Context } from "../../context/index.ts"; +import type { SqliteExecutor, SqliteValue } from "../../sqlite/index.ts"; +import { checkName } from "../contracts/internal.ts"; +import type { Claimant } from "./claims.ts"; +import type { WhereNode } from "./expression.ts"; +import type { EncodedRow } from "./rows.ts"; +import { type Order, TableSql, tableSqlName } from "./sql.ts"; +import type { Step } from "./step.ts"; +import type { ChordStore } from "./store.ts"; +import type { TableDefinition } from "./table.ts"; +import { prepareTables } from "./table-migration.ts"; +import type { TableWrite } from "./table-writes.ts"; + +/** A stored row as reads decode it. */ +export type DecodedRow = { readonly [column: string]: unknown }; + +/** + * The tables of one owner name in a store: each declared once (its SQL name `":
"` claimed for + * `claimant`), then prepared together (created at their newest version or migrated, as `self.table`'s are). + */ +export class StoreOwnerTables { + readonly store: ChordStore; + readonly owner: string; + readonly #claimant: Claimant; + readonly #tables: OwnerTable[] = []; + + /** `owner` is a restricted name (lowercase `[a-z0-9@/._~-]`): it becomes the first part of SQL names. */ + constructor(store: ChordStore, owner: string, claimant: Claimant) { + checkName(owner, "An owner name"); + this.store = store; + this.owner = owner; + this.#claimant = claimant; + } + + /** Declares `definition` under the owner, claiming its SQL name (held by another: `invalid/args`). */ + declare(definition: TableDefinition): OwnerTable { + const table = new OwnerTable(definition, tableSqlName(this.owner, definition.name)); + this.store.claims.table(table.sql.table, this.#claimant); + this.#tables.push(table); + return table; + } + + /** Brings every declared table to its definition: created fresh, or migrated eagerly in their commits. */ + prepare(context: Context): Promise { + const tables = this.#tables.map(({ definition, sql }) => ({ definition, table: sql.table })); + const fence = { admit: () => true, adopt: () => {} }; + return prepareTables(tables, this.store, fence, this.owner, context); + } +} + +/** One declared table of an owner: its rows encoded and written in a step, its reads by key and by query. */ +export class OwnerTable { + readonly definition: TableDefinition; + readonly sql: TableSql; + + constructor(definition: TableDefinition, table: string) { + this.definition = definition; + this.sql = new TableSql(table, definition.columns, definition.rows.physical, definition.key); + } + + /** `row` checked and encoded for a `put` (a trust boundary, rule 8: at staging, before the queue). */ + encode(row: unknown): EncodedRow { + return this.definition.rows.put(row, this.definition.version); + } + + /** + * Writes `encoded` in `step` (an upsert) and returns the `put` as recorded; `stamp` names an `int` column that takes + * the step's seq (Durable's `entries.commit_seq`), in the row and in the record. + */ + async put(step: Step, encoded: EncodedRow, stamp?: string): Promise { + const { version } = this.definition; + const { table } = this.sql; + if (stamp === undefined) { + await step.run(this.sql.put, ...encoded.params); + return { kind: "put", table, row: encoded.row, version, columns: encoded.columns }; + } + const params = [...encoded.params]; + params[this.definition.rows.physical.indexOf(stamp)] = step.seq; + await step.run(this.sql.put, ...params); + return { kind: "put", table, row: { ...encoded.row, [stamp]: step.seq }, version, columns: encoded.columns }; + } + + /** The row with the key `params` (in key order), decoded. */ + async get( + executor: Pick, + params: readonly SqliteValue[], + context: Context, + ): Promise { + const stored = await executor.get<{ [column: string]: SqliteValue }>(this.sql.get, ...params); + return stored === undefined ? undefined : this.definition.rows.decode(stored, context); + } + + /** + * A read compiled once (the store's query vocabulary rendered to SQL): `columns` of the rows `where` matches in + * `order`. The values in `where` only shape it: each is a `?` the caller binds in the order the conditions take + * them, then the limit. Adapters keep their fixed reads so: no SQL is rendered per call. + */ + prepare(columns: readonly string[], where: WhereNode | undefined, order: Order): string { + return this.sql.select(where, order, undefined, 0, columns).sql; + } + + /** `prepare` for the first row only: the limit is a literal `1` (a bound one costs SQLite a plan step per call). */ + prepareFirst(columns: readonly string[], where: WhereNode | undefined, order: Order): string { + return this.sql.select(where, order, undefined, 1, columns).sql.replace(/LIMIT \?$/, "LIMIT 1"); + } + + /** + * The version of `json(...)` column `column`: at 1 every stored cell is at 1 (none is older, a newer one fails the + * owner when its table is prepared), so a read may leave the version column out. + */ + jsonVersion(column: string): number { + const found = this.definition.column(column, "jsonVersion"); + if (found.type !== "json") throw new Error(`${this.definition.name}: column "${column}" is no json(...) column`); + return found.json.version; + } + + /** A point lookup compiled once: `columns` of the row whose key the caller binds, in key order. */ + prepareGet(columns: readonly string[]): string { + return this.sql.getColumns(columns); + } + + /** + * The read of `json(...)` column `column`'s stored cells (`text` written at `version`, migrated on access), made + * once: an adapter selecting the cell itself reads it so. + */ + reader(column: string): (text: string, version: number, context: Context) => unknown { + const found = this.definition.column(column, "reader"); + if (found.type !== "json") throw new Error(`${this.definition.name}: column "${column}" is no json(...) column`); + // At version 1 a cell is the JSON it was written as: no version to compare, nothing to migrate. + if (found.json.version === 1) return (text) => JSON.parse(text) as unknown; + const what = `${this.sql.table}.${column}`; + return (text, version, context) => found.json.read(text, version, what, context); + } + + /** Up to `limit` rows matching `where` in `order`, decoded. */ + async select( + executor: Pick, + where: WhereNode | undefined, + order: Order, + limit: number, + context: Context, + ): Promise { + const statement = this.sql.select(where, order, undefined, limit); + const stored = await executor.all<{ [column: string]: SqliteValue }>(statement.sql, ...statement.params); + return stored.map((row) => this.definition.rows.decode(row, context)); + } +} diff --git a/packages/chord/src/next/store/allocator.ts b/packages/chord/src/next/store/allocator.ts new file mode 100644 index 000000000..6ff6d120a --- /dev/null +++ b/packages/chord/src/next/store/allocator.ts @@ -0,0 +1,33 @@ +// The store's id allocator (slice 6b D3, §1.2): one counter for every id in the store (Durable's `mintId`, Chord's +// document ids), synchronous, raised by explicit ids, saturating at `Number.MAX_SAFE_INTEGER`. + +/** The first id the allocator gives: id 1 is Durable's root conversation. */ +export const FIRST_ID = 2; + +/** + * The allocator: `value` is the next id it gives. An id raised to `Number.MAX_SAFE_INTEGER` is accepted, the value + * stays there, and every later `mint` throws: no id past the safe range (it would stop counting exactly and repeat). + */ +export class IdAllocator { + #next: number; + + constructor(next: number) { + this.#next = next; + } + + /** The next id `mint` gives; what each commit persists. */ + get value(): number { + return this.#next; + } + + /** Takes the next id; throws "ID space is exhausted" once saturated (Durable's message). */ + mint(): number { + if (this.#next >= Number.MAX_SAFE_INTEGER) throw new Error("ID space is exhausted"); + return this.#next++; + } + + /** Raises the counter above `id`, an id written explicitly (Durable mints ahead), saturating. */ + raise(id: number): void { + if (id >= this.#next) this.#next = Math.min(id + 1, Number.MAX_SAFE_INTEGER); + } +} diff --git a/packages/chord/src/next/store/applier.ts b/packages/chord/src/next/store/applier.ts index f6e55e214..a1bc6c862 100644 --- a/packages/chord/src/next/store/applier.ts +++ b/packages/chord/src/next/store/applier.ts @@ -1,25 +1,37 @@ -// The applier (slice 6a §13): recorded writes applied as they were recorded, inside a backend transaction, checked -// against the store's own records (a trust boundary: archives, later a replica's batches). One code path for import, -// bootstrap and, from slice 6b, batch apply; 6a applies the kinds an archive holds. +// The applier (slice 6a §13, slice 6b §1.6): recorded writes applied as they were recorded, inside a backend +// transaction, checked against the store's own records (a trust boundary: archives, a replica's batches). One code +// path for import, bootstrap and batch apply, for every write kind; a recorded count the store does not reproduce +// stops applying (`Diverged`, S17). import type { SqliteValue } from "../../sqlite/index.ts"; import { checkName } from "../contracts/internal.ts"; import { invalid } from "../data/internal.ts"; -import type { ArchivePart } from "./archive.ts"; import type { TableRecord } from "./catalog.ts"; import { COLUMN_NAME, type ColumnRecord } from "./columns.ts"; -import { insertDocument } from "./engine.ts"; +import { + type DocumentContent, + type DocumentRecord, + type DocumentWrite, + insertDocument, + insertRevision, + type RevisionContent, + readDocument, + retireRecord, + writeContent, +} from "./engine.ts"; +import type { ExprNode, SetNode, WhereNode } from "./expression.ts"; import { checkScalar, ROW_VERSION, toSqlite, versionColumn } from "./rows.ts"; -import { recordedPut } from "./sql.ts"; +import { recordedPut, TableSql } from "./sql.ts"; import type { Step } from "./step.ts"; import { INDEX_NAME } from "./table.ts"; -import { applyIndex, applyTable, type TableWrite } from "./table-writes.ts"; +import { applyAlter, applyDropIndex, applyIndex, applyTable, type StoredKey, type TableWrite } from "./table-writes.ts"; +import type { StoreWrite } from "./writes.ts"; -/** A recorded write the applier takes: the writes of an archive. */ -export type AppliedWrite = Exclude; +/** A recorded write the store does not reproduce (a count, a revision): applying stops, the store diverged (S17). */ +export class Diverged extends Error {} -/** Applies `write` in `step`; a write the records refuse throws `invalid/args` (`what` names its place). */ -export async function applyWrite(step: Step, write: AppliedWrite, what: string): Promise { +/** Applies `write` in `step`; a write the records refuse throws `invalid/args`, one they do not reproduce `Diverged`. */ +export async function applyWrite(step: Step, write: StoreWrite, what: string): Promise { switch (write.kind) { case "table": checkTable(write, what); @@ -29,29 +41,238 @@ export async function applyWrite(step: Step, write: AppliedWrite, what: string): checkIndex(recordOf(step, write.table, what), write, what); await applyIndex(step, write); return; + case "dropIndex": + if (!Object.hasOwn(recordOf(step, write.table, what).indexes, write.name)) { + throw invalid("args", `${what}: no index ${JSON.stringify(write.name)}`); + } + await applyDropIndex(step, write); + return; + case "alter": + checkAlter(recordOf(step, write.table, what), write, what); + await applyAlter(step, write); + return; case "put": { const record = recordOf(step, write.table, what); const { sql, physical } = recordedPut(write.table, record.columns); await step.run(sql, ...putParams(record, write, physical, what)); return; } - case "document.create": { - let text: string; - try { - text = JSON.stringify(write.base.value); - } catch (error) { - throw invalid("args", `${what}: the base is not JSON (${String(error)})`); - } - await insertDocument(step, write.record, text); - return; - } + case "update": + case "delete": + case "updateWhere": + case "deleteWhere": + return applyCounted(step, write, what); + default: + return applyDocument(step, write, what); } } -/** A recorded table's SQL name (three names joined by `:`), columns, key and json versions, by the name rules. */ +/** Applies a document write: a creation, a change or a retirement on its recorded revision, an archive's revision. */ +async function applyDocument(step: Step, write: DocumentWrite, what: string): Promise { + switch (write.kind) { + case "document.create": + await insertDocument(step, write.record, write.base === undefined ? undefined : json(write.base.value, what)); + return; + case "document.revision": { + const { content } = write; + await insertRevision(step, write.id, write.seq, content.kind, content.version, contentText(content, what)); + return; + } + case "document.change": { + const record = await liveAt(step, write.id, write.revision, what); + const { content } = write; + const text = contentText(content, what); + await writeContent(step, write.id, record.history, write.revision, content, text); + return; + } + case "document.retire": + await retireRecord(step, await liveAt(step, write.id, write.revision, what)); + return; + } +} + +/** A content's stored text: a base's value or a delta's ops, checked as JSON. */ +function contentText(content: DocumentContent | RevisionContent, what: string): string { + return json(content.kind === "base" ? content.value : content.ops, what); +} + +/** The live record `id` at `revision`; another revision, or none, is a divergence. */ +async function liveAt(step: Step, id: number, revision: number, what: string): Promise { + const record = await readDocument(step, id); + if (record === undefined || record.retiredAt !== undefined || record.revision !== revision) { + throw new Diverged(`${what}: document ${id} is not live at revision ${revision}`); + } + return record; +} + +/** `value` serialized; one `JSON.stringify` cannot encode is refused. */ +function json(value: unknown, what: string): string { + let text: string | undefined; + try { + text = JSON.stringify(value); + } catch (error) { + throw invalid("args", `${what}: not JSON (${String(error)})`); + } + if (text === undefined) throw invalid("args", `${what}: not JSON`); + return text; +} + +/** + * Applies a counted table write: its statement rendered from the record's columns and the recorded `key`, `where` and + * `set` (checked: only recorded columns, well-formed nodes), and its count compared with the recorded one. + */ +async function applyCounted( + step: Step, + write: Extract, + what: string, +): Promise { + const record = recordOf(step, write.table, what); + const columns = Object.entries(record.columns).map(([name, column]) => ({ name, type: column.type })); + const sql = new TableSql(write.table, columns, [], record.key); + let statement: { readonly sql: string; readonly params: SqliteValue[] }; + switch (write.kind) { + case "update": + statement = sql.update( + setOf(record, write, what), + keyParams(record, write.key, what), + versionsOf(record, write, what), + ); + break; + case "delete": + statement = { sql: sql.delete, params: keyParams(record, write.key, what) }; + break; + case "updateWhere": + checkWhere(record, write.where, what); + statement = sql.updateWhere(write.where, setOf(record, write, what), versionsOf(record, write, what)); + break; + case "deleteWhere": + checkWhere(record, write.where, what); + statement = sql.deleteWhere(write.where); + break; + } + const count = await step.runCounted(statement.sql, ...statement.params); + if (count !== write.count) throw new Diverged(`${what}: changed ${count} rows, recorded ${write.count}`); +} + +/** The recorded key as parameters in key order: exactly the key columns, each of its type. */ +function keyParams(record: TableRecord, key: StoredKey, what: string): SqliteValue[] { + if (Object.keys(key).length !== record.key.length) throw invalid("args", `${what}: not a key of ${record.table}`); + return record.key.map((name) => { + if (!Object.hasOwn(key, name)) throw invalid("args", `${what}: key column ${JSON.stringify(name)} is missing`); + return toSqlite(checkScalar(scalarType(record.columns[name]!), key[name], `${what}: key`)); + }); +} + +/** A recorded `set` with the migration stamp `$v` when it carries one; key columns never change (rule 13). */ +function setOf( + record: TableRecord, + write: Extract, + what: string, +): SetNode { + const { set } = write; + if (typeof set !== "object" || set === null || Object.keys(set).length === 0) { + throw invalid("args", `${what}: an empty or malformed set`); + } + for (const [column, node] of Object.entries(set)) { + if (!Object.hasOwn(record.columns, column) || record.key.includes(column)) { + throw invalid("args", `${what}: set writes ${JSON.stringify(column)}, no non-key column`); + } + checkExpr(record, node, what); + } + return write.version === undefined ? set : { ...set, [ROW_VERSION]: write.version }; +} + +/** The `json(...)` versions a `set` stamps: the recorded ones, of `json(...)` columns it sets, up to theirs. */ +function versionsOf( + record: TableRecord, + write: Extract, + what: string, +): Map { + const versions = new Map(); + for (const [name, column] of Object.entries(record.columns)) { + if (column.type !== "json" || !Object.hasOwn(write.set, name)) continue; + // A clear stamps `NULL` whatever the version; a value stamps its recorded version. + const version = write.columns[name] ?? column.version!; + if (!Number.isSafeInteger(version) || version < 1 || version > column.version!) { + throw invalid("args", `${what}: no readable version for ${JSON.stringify(name)}`); + } + versions.set(name, version); + } + return versions; +} + +/** A scalar column's type; `json(...)` columns are not scalars. */ +function scalarType(column: ColumnRecord): "str" | "int" | "num" | "bool" { + return column.type === "json" ? "str" : column.type; +} + +const COMPARISONS = ["eq", "ne", "lt", "lte", "gt", "gte"] as const; + +/** A recorded condition: well-formed nodes over recorded scalar columns (and the row's `$v`). */ +function checkWhere(record: TableRecord, node: unknown, what: string): asserts node is WhereNode { + const refuse = (): never => { + throw invalid("args", `${what}: a malformed condition`); + }; + if (typeof node !== "object" || node === null || Object.keys(node).length !== 1) return refuse(); + const [[op, operand]] = Object.entries(node) as [[string, unknown]]; + const column = (name: unknown): void => { + if (name === ROW_VERSION) return; + if (typeof name !== "string" || record.columns[name] === undefined || record.columns[name]!.type === "json") { + refuse(); + } + }; + if (op === "and" || op === "or") { + if (!Array.isArray(operand)) return refuse(); + for (const each of operand) checkWhere(record, each, what); + } else if (op === "not") checkWhere(record, operand, what); + else if (op === "isNull") column(operand); + else if (op === "inArray" || op === "prefix" || (COMPARISONS as readonly string[]).includes(op)) { + if (!Array.isArray(operand) || operand.length !== 2) return refuse(); + column(operand[0]); + const values = op === "inArray" ? operand[1] : [operand[1]]; + if (!Array.isArray(values)) return refuse(); + for (const value of values) if (!isScalar(value) || (op === "prefix" && typeof value !== "string")) refuse(); + } else refuse(); +} + +const OPERATIONS = ["add", "sub", "max", "min", "concat", "coalesce"] as const; + +/** A recorded expression: scalars, `null`, JSON literals, recorded columns and the vocabulary's operations. */ +function checkExpr(record: TableRecord, node: unknown, what: string): asserts node is ExprNode { + if (node === null || isScalar(node)) return; + const refuse = (): never => { + throw invalid("args", `${what}: a malformed expression`); + }; + if (typeof node !== "object" || Object.keys(node).length !== 1) return refuse(); + const [[op, operand]] = Object.entries(node) as [[string, unknown]]; + if (op === "col") { + if (typeof operand !== "string" || !Object.hasOwn(record.columns, operand)) refuse(); + return; + } + if (op === "json") return; + if (!Array.isArray(operand)) return refuse(); + if ((OPERATIONS as readonly string[]).includes(op) && operand.length === 2) { + for (const each of operand) checkExpr(record, each, what); + } else if (op === "replacePrefix" && operand.length === 3) { + checkExpr(record, operand[0], what); + if (typeof operand[1] !== "string" || typeof operand[2] !== "string") refuse(); + } else if (op === "when" && operand.length === 3) { + checkWhere(record, operand[0], what); + checkExpr(record, operand[1], what); + checkExpr(record, operand[2], what); + } else refuse(); +} + +function isScalar(value: unknown): boolean { + return ( + typeof value === "string" || typeof value === "boolean" || (typeof value === "number" && Number.isFinite(value)) + ); +} + +/** A recorded table's SQL name (two names joined by `:`, R2), columns, key and json versions, by the name rules. */ function checkTable(write: Extract, what: string): void { const parts = write.table.split(":"); - if (parts.length !== 3) throw invalid("args", `${what}: table ${JSON.stringify(write.table)}: not kind:type:table`); + if (parts.length !== 2) throw invalid("args", `${what}: table ${JSON.stringify(write.table)}: not owner:table`); try { for (const part of parts) checkName(part, `${what}: table name part`); } catch (error) { @@ -80,6 +301,30 @@ function checkColumn(column: ColumnRecord, what: string): void { checkScalar(column.type, column.default, `${what}: the default`); } +/** A recorded column edit of `record`'s next version or its own (the edits after a data step). */ +function checkAlter(record: TableRecord, write: Extract, what: string): void { + if (write.version !== record.version && write.version !== record.version + 1) { + throw invalid("args", `${what}: an edit of version ${write.version} on version ${record.version}`); + } + const { edit } = write; + const has = (name: string): boolean => Object.hasOwn(record.columns, name); + const fresh = (name: string): boolean => + COLUMN_NAME.test(name) && !Object.keys(record.columns).some((each) => each.toLowerCase() === name.toLowerCase()); + const indexed = (name: string): boolean => + record.key.includes(name) || Object.values(record.indexes).some((columns) => columns.includes(name)); + const valid = + edit.kind === "rename" + ? has(edit.from) && fresh(edit.to) + : edit.kind === "add" + ? fresh(edit.name) && (edit.column.optional === true || edit.column.default !== undefined) + : edit.kind === "drop" + ? has(edit.name) && !indexed(edit.name) + : has(edit.name) && record.columns[edit.name]!.type === "json" && edit.column.type === "json"; + if (!valid) throw invalid("args", `${what}: an ${edit.kind} edit the table cannot take`); + if (edit.kind === "add" || edit.kind === "json") + checkColumn(edit.column, `${what}: column ${JSON.stringify(edit.name)}`); +} + /** `names` are distinct, recorded, non-`opt` `str` or `int` columns. */ function checkOrderColumns( columns: { readonly [name: string]: ColumnRecord }, @@ -152,11 +397,7 @@ function putParams( } const version = versions[name]; if (version === undefined || version > column.version!) throw invalid("args", `${where}: no readable version`); - try { - cells.set(name, JSON.stringify(value)); - } catch (error) { - throw invalid("args", `${where}: not JSON (${String(error)})`); - } + cells.set(name, json(value, where)); cells.set(versionColumn(name), version); } return physical.map((name) => (name === ROW_VERSION ? version : (cells.get(name) ?? null))); diff --git a/packages/chord/src/next/store/apply-batch.ts b/packages/chord/src/next/store/apply-batch.ts new file mode 100644 index 000000000..be4be9ad2 --- /dev/null +++ b/packages/chord/src/next/store/apply-batch.ts @@ -0,0 +1,99 @@ +// Batch apply (slice 6b §1.6, S5, S17): a replica takes commits `from..to` as an outbox gave them, in one backend +// transaction through the store's commit queue, each write applied with its commit's seq by the applier (a trust +// boundary), `lastSeq` and the allocator set from the last commit. Internal: the sync layer's entry point. + +import type { Context } from "../../context/index.ts"; +import type { SqliteExecutor } from "../../sqlite/index.ts"; +import { invalid } from "../data/internal.ts"; +import { IdAllocator } from "./allocator.ts"; +import { applyWrite, Diverged } from "./applier.ts"; +import { checkWrite } from "./archive.ts"; +import { type TableRecord, writeCatalog } from "./catalog.ts"; +import { OUTBOX_FORMAT, type OutboxCommit } from "./outbox.ts"; +import { TABLES } from "./schema.ts"; +import { Step } from "./step.ts"; +import type { ChordStore, Replayed } from "./store.ts"; + +/** + * Applies `commits` (from an outbox, in seq order) to `store`: refused while an owner holds a claim on it, once it + * diverged, or when the first commit does not follow its `lastSeq` or one does not follow the one before; a write the + * store's records refuse rolls the whole batch back (`invalid/args`); a recorded count the store does not reproduce + * rolls it back and marks the store diverged, so every later batch is refused. Commits written to the store's outbox + * too when it has one. + */ +export function applyBatch(store: ChordStore, commits: readonly OutboxCommit[], context: Context): Promise { + let checked: OutboxCommit[]; + try { + checked = commits.map((commit, index) => checkCommit(commit, `batch: commit ${index}`, context)); + } catch (error) { + return Promise.reject(error); + } + return store.replay( + (executor, catalog) => replay(store, checked, executor, catalog), + (error) => error instanceof Diverged, + ); +} + +/** One commit's envelope and writes, checked (a trust boundary). */ +function checkCommit(commit: OutboxCommit, what: string, context: Context): OutboxCommit { + const { seq, nextId, writes } = commit as Partial; + if (!Number.isSafeInteger(seq) || !Number.isSafeInteger(nextId) || !Array.isArray(writes)) { + throw invalid("args", `${what}: not an outbox commit`); + } + return { + seq: seq!, + nextId: nextId!, + writes: writes.map((write, index) => checkWrite(write, `${what}: write ${index}`, context)), + }; +} + +/** The batch inside the store's turn: seqs continuous from `lastSeq`, allocator values never falling. */ +async function replay( + store: ChordStore, + commits: readonly OutboxCommit[], + executor: SqliteExecutor, + catalog: ReadonlyMap, +): Promise { + const tables = new Map(catalog); + const changed = new Map(); + let lastSeq = store.lastSeq; + let nextId = store.nextId; + for (const [index, commit] of commits.entries()) { + const what = `batch: commit ${index} (seq ${commit.seq})`; + if (commit.seq !== lastSeq + 1) throw invalid("args", `${what}: does not follow seq ${lastSeq}`); + if (commit.nextId < nextId) throw invalid("args", `${what}: its next id ${commit.nextId} is below ${nextId}`); + const step = new Step(executor, commit.seq, new IdAllocator(commit.nextId), tables); + for (const [position, write] of commit.writes.entries()) { + const where = `${what}: write ${position} (${write.kind})`; + if (write.kind === "document.create") checkCreated(write.record.id, commit.nextId, where); + await applyWrite(step, write, where); + } + await writeCatalog(step); + for (const [name, record] of step.tables) { + tables.set(name, record); + changed.set(name, record); + } + if (store.recordsOutbox) { + await executor.run( + `INSERT INTO ${TABLES.outbox} (seq, format, next_id, writes) VALUES (?, ?, ?, ?)`, + commit.seq, + OUTBOX_FORMAT, + commit.nextId, + JSON.stringify(commit.writes), + ); + } + lastSeq = commit.seq; + nextId = commit.nextId; + } + await executor.run(`UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ? WHERE singleton = 1`, lastSeq, nextId); + return { lastSeq, nextId, tables: changed }; +} + +/** + * A created document's id is below its commit's next id (the allocator never mints it again), or equal to a saturated + * next id (`Number.MAX_SAFE_INTEGER` may be written, as import allows). + */ +function checkCreated(id: number, nextId: number, what: string): void { + if (id < nextId || (nextId === Number.MAX_SAFE_INTEGER && id === nextId)) return; + throw invalid("args", `${what}: id ${id} is not below the commit's next id ${nextId}`); +} diff --git a/packages/chord/src/next/store/archive.ts b/packages/chord/src/next/store/archive.ts index 963cfae48..1db668048 100644 --- a/packages/chord/src/next/store/archive.ts +++ b/packages/chord/src/next/store/archive.ts @@ -1,6 +1,6 @@ -// Archives (slice 6a §13; decisions "Export archives are operations", "Writes carry their versions; archives are -// lossless"): the parts an export yields and an import takes, their order, and the one compiled schema every -// imported part is checked against first. +// Archives and recorded commits as they cross a trust boundary (slice 6a §13, slice 6b §1.6; decisions "Export archives +// are operations", "Writes carry their versions; archives are lossless"): the parts an export yields and an import +// takes, their order, and the compiled schemas every imported part and every applied write is checked against first. import Type, { type TInteger } from "typebox"; import type { Context } from "../../context/index.ts"; @@ -8,28 +8,29 @@ import { arr, bool, int, lit, num, obj, opt, str, union, unknown } from "../cont import { invalid, validatorFor } from "../data/internal.ts"; import type { DocumentWrite } from "./engine.ts"; import type { TableWrite } from "./table-writes.ts"; +import type { StoreWrite } from "./writes.ts"; /** The archive format `export` writes; `import` takes this one only. */ export const ARCHIVE_FORMAT = 1; -/** The first part: the format, the store's schema version, its `lastSeq` and the next document id (rule 24). */ +/** The first part: the format, the store's schema version, its `lastSeq` and the allocator's next id (rule 24, D3). */ export type ArchiveManifest = { readonly kind: "manifest"; readonly format: number; readonly schema: number; readonly lastSeq: number; - readonly nextDocumentId: number; + readonly nextId: number; }; /** * One part of an archive, in this order: the manifest; per table (by SQL name) its `table` write, its `put`s and its - * `index` writes; the live documents' `document.create`s (by id, the current value as the base); `end` with the - * count of the parts between. + * `index` writes; per document record (by id, live and retired) its `document.create` without a base, then its kept + * revisions in seq order; `end` with the count of the parts between. */ export type ArchivePart = | ArchiveManifest | Extract - | Extract + | Extract | { readonly kind: "end"; readonly writes: number }; /** An integer from `min` up to 2^53 - 1: seqs, ids, versions and counts stay exact (a seq after 2^53 would repeat). */ @@ -47,54 +48,113 @@ const column = obj({ fingerprint: opt(str()), }); +const versions = Type.Record(Type.String(), version); + +const TABLE = obj({ + kind: lit("table"), + table: str(), + version, + state: union([lit("ready"), lit("migrating")]), + columns: Type.Record(Type.String(), column), + key: arr(str()), + fingerprint: str(), +}); + +const PUT = obj({ + kind: lit("put"), + table: str(), + row: Type.Record(Type.String(), unknown()), + version, + columns: versions, +}); + +const INDEX = obj({ kind: lit("index"), table: str(), name: str(), columns: arr(str()) }); + +const RECORD = obj({ + id: safe(1), + owner: str(), + namespace: str(), + kind: str(), + // Absent for a singleton. + key: opt(str()), + history: union([lit("none"), lit("latest"), lit("rewindable")]), + createdAt: safe(1), + retiredAt: opt(safe(1)), + meta: opt(str()), + revision: safe(1), + version, + fingerprint: str(), +}); + +const BASE = obj({ version, fingerprint: str(), value: unknown() }); + +/** A delta's ops; their shapes are checked where they are applied. */ +const ops = arr(unknown()); + +const REVISION = obj({ + kind: lit("document.revision"), + id: safe(1), + seq: safe(1), + content: union([obj({ kind: lit("base"), version, value: unknown() }), obj({ kind: lit("delta"), version, ops })]), +}); + const PART = union([ - obj({ - kind: lit("manifest"), - format: safe(1), - schema: safe(1), - lastSeq: safe(0), - nextDocumentId: safe(1), - }), - obj({ - kind: lit("table"), - table: str(), - version, - state: union([lit("ready"), lit("migrating")]), - columns: Type.Record(Type.String(), column), - key: arr(str()), - fingerprint: str(), - }), - obj({ - kind: lit("put"), - table: str(), - row: Type.Record(Type.String(), unknown()), - version, - columns: Type.Record(Type.String(), version), - }), - obj({ kind: lit("index"), table: str(), name: str(), columns: arr(str()) }), - obj({ - kind: lit("document.create"), - record: obj({ - id: safe(1), - namespace: str(), - owner: str(), - kind: str(), - // Absent for a singleton. - key: opt(str()), - history: union([lit("latest"), lit("rewindable")]), - createdAt: safe(1), - meta: opt(str()), - revision: safe(1), - version, - fingerprint: str(), - }), - base: obj({ version, fingerprint: str(), value: unknown() }), - }), + obj({ kind: lit("manifest"), format: safe(1), schema: safe(1), lastSeq: safe(0), nextId: safe(1) }), + TABLE, + PUT, + INDEX, + obj({ kind: lit("document.create"), record: RECORD }), + REVISION, obj({ kind: lit("end"), writes: safe(0) }), ]); +const count = safe(0); + +const edit = union([ + obj({ kind: lit("rename"), from: str(), to: str() }), + obj({ kind: lit("add"), name: str(), column }), + obj({ kind: lit("drop"), name: str() }), + obj({ kind: lit("json"), name: str(), column }), +]); + +const key = Type.Record(Type.String(), union([str(), num(), bool()])); + +/** One recorded write of a commit (a batch's); `where` and `set` nodes are checked by the applier. */ +const WRITE = union([ + TABLE, + PUT, + INDEX, + obj({ kind: lit("dropIndex"), table: str(), name: str() }), + obj({ kind: lit("alter"), table: str(), version, edit }), + obj({ kind: lit("update"), table: str(), key, set: unknown(), columns: versions, count, version: opt(version) }), + obj({ kind: lit("delete"), table: str(), key, count }), + obj({ + kind: lit("updateWhere"), + table: str(), + where: unknown(), + set: unknown(), + columns: versions, + count, + version: opt(version), + }), + obj({ kind: lit("deleteWhere"), table: str(), where: unknown(), count }), + obj({ kind: lit("document.create"), record: RECORD, base: BASE }), + obj({ + kind: lit("document.change"), + id: safe(1), + revision: safe(1), + content: union([ + obj({ kind: lit("base"), version, fingerprint: str(), value: unknown() }), + obj({ kind: lit("delta"), version, ops }), + ]), + }), + obj({ kind: lit("document.retire"), id: safe(1), revision: safe(1) }), +]); + const PART_SCHEMA = { schema: PART }; +const WRITE_SCHEMA = { schema: WRITE }; + /** Checks one imported part against the archive schema (a trust boundary); `invalid/args` naming its position. */ export function checkPart(part: unknown, position: number, context: Context): ArchivePart { context.telemetry.metrics?.counter("chord.data.validations").add(1); @@ -104,3 +164,11 @@ export function checkPart(part: unknown, position: number, context: Context): Ar // The check above establishes the type. return part as ArchivePart; } + +/** Checks one write of a batch's commit against the write schema (a trust boundary); `what` names its place. */ +export function checkWrite(write: unknown, what: string, context: Context): StoreWrite { + context.telemetry.metrics?.counter("chord.data.validations").add(1); + if (!validatorFor(WRITE_SCHEMA, context).Check(write)) throw invalid("args", `${what}: not a store write`); + // The check above establishes the type, the nodes aside (the applier checks them). + return write as StoreWrite; +} diff --git a/packages/chord/src/next/store/claims.ts b/packages/chord/src/next/store/claims.ts new file mode 100644 index 000000000..2f753f18f --- /dev/null +++ b/packages/chord/src/next/store/claims.ts @@ -0,0 +1,57 @@ +// Claims (slice 6b "owner names and claims", "claims per table and per document namespace"): what can collide in a +// store is claimed once, in memory: a table by its SQL name, documents by `(owner, namespace)`. A claim is held by its +// claimant (an object, a `StoreStorage`) until its end releases every claim it holds. While a batch applies, nothing +// is claimed: batch apply and owners exclude each other both ways. + +import { invalid } from "../data/internal.ts"; + +/** Who holds claims: named in the refusal of a second claim. */ +export type Claimant = { readonly label: string }; + +/** A store's claims, by key; one claimant per key. */ +export class Claims { + readonly #holders = new Map(); + readonly #keys = new Map(); + /** Whether a batch applies now: no claim is taken meanwhile. */ + #locked = false; + + /** Whether any claimant holds a claim (batch apply is refused then: its memory would go stale). */ + get held(): boolean { + return this.#keys.size > 0; + } + + /** Claims table `table` (its SQL name) for `claimant`; held by another: `invalid/args`. */ + table(table: string, claimant: Claimant): void { + this.#claim(`table ${JSON.stringify(table)}`, claimant); + } + + /** Claims the documents of `owner` in `namespace` for `claimant`; held by another: `invalid/args`. */ + documents(owner: string, namespace: string, claimant: Claimant): void { + this.#claim(`documents ${JSON.stringify([owner, namespace])}`, claimant); + } + + /** Refuses every claim (`invalid/args`) until the returned unlock: a batch applies meanwhile. */ + lock(): () => void { + this.#locked = true; + return () => { + this.#locked = false; + }; + } + + /** Releases every claim `claimant` holds. */ + release(claimant: Claimant): void { + for (const key of this.#keys.get(claimant) ?? []) this.#holders.delete(key); + this.#keys.delete(claimant); + } + + #claim(key: string, claimant: Claimant): void { + if (this.#locked) throw invalid("args", `store: ${key} cannot be claimed while a batch applies`); + const holder = this.#holders.get(key); + if (holder === claimant) return; + if (holder !== undefined) throw invalid("args", `store: ${key} is claimed by ${holder.label}`); + this.#holders.set(key, claimant); + const keys = this.#keys.get(claimant); + if (keys === undefined) this.#keys.set(claimant, [key]); + else keys.push(key); + } +} diff --git a/packages/chord/src/next/store/engine.ts b/packages/chord/src/next/store/engine.ts index b5fbc8f80..b30b70b2a 100644 --- a/packages/chord/src/next/store/engine.ts +++ b/packages/chord/src/next/store/engine.ts @@ -1,6 +1,7 @@ -// The documents engine (S13, slice 6a §8): one record per incarnation of an address, revisions made of a base and -// later deltas; its writes inside the commit step, checked by the writes themselves (the partial unique index on the -// live address, the counted conditional `UPDATE` on the record's revision), and its plain reads. +// The documents engine (S13, slice 6a §8, slice 6b §1.5): one record per incarnation of an address, revisions made of +// bases and deltas, a history rule per record (R1); its writes inside the commit step, checked by the writes +// themselves (the partial unique index on the live address, the counted conditional `UPDATE` on the record's +// revision), and its reads, of the current state or at a point. import { apply, type Op } from "../../delta/index.ts"; import type { JsonValue } from "../../json.ts"; @@ -9,23 +10,33 @@ import { TABLES } from "./schema.ts"; import type { StagedWrite, Step } from "./step.ts"; /** - * A document's address: the adapter's namespace, the owner string, the definition's name and, for a keyed document, its - * key; a singleton has none (as Durable's addresses). The column keeps `""` for it, which a keyed document's empty key - * shares (its definition says which it is): a record read back gives no key for either. + * A document's address: the owner name, the adapter's namespace (claimed with the owner: Chord's `chord`, `chord.jobs`, + * `chord.job-states`; Durable's scopes), the kind and, for a keyed document, its key; a singleton has none. The column + * keeps `""` for it, which a keyed document's empty key shares (its definition says which it is). */ export type DocumentAddress = { - readonly namespace: string; readonly owner: string; + readonly namespace: string; readonly kind: string; readonly key?: string; }; -/** A stored record: one incarnation of an address. Chord keeps only the latest revision (`history: "latest"`). */ +/** + * What a record keeps (R1), applied at a new base and at retirement: `none` (Chord) drops older revisions at a new base + * and the row and its revisions at retirement; `latest` drops older revisions at a new base and its revisions at + * retirement, keeping the row; `rewindable` keeps everything. + */ +export type DocumentHistory = "none" | "latest" | "rewindable"; + +/** A stored record: one incarnation of an address, live from `createdAt` until `retiredAt` (half-open). */ export type DocumentRecord = DocumentAddress & { readonly id: number; - readonly history: "latest" | "rewindable"; + readonly history: DocumentHistory; /** The seq that created it. */ readonly createdAt: number; + /** The seq that retired it, once retired. */ + readonly retiredAt?: number; + /** The adapter's details (Durable: its fork policy and history). */ readonly meta?: string; /** Counts its changes: a change names the revision it was prepared on. */ readonly revision: number; @@ -42,130 +53,165 @@ export type DocumentContent = | ({ readonly kind: "base" } & DocumentBase) | { readonly kind: "delta"; readonly version: number; readonly ops: readonly Op[] }; -/** The engine's writes as the outbox and archives record them. */ +/** A stored revision's content: a base's value or a delta's ops, at a version (a revision keeps no fingerprint). */ +export type RevisionContent = + | { readonly kind: "base"; readonly version: number; readonly value: JsonValue } + | { readonly kind: "delta"; readonly version: number; readonly ops: readonly Op[] }; + +/** One stored revision as archives carry it: its document, its seq, its content. */ +export type DocumentRevision = { readonly id: number; readonly seq: number; readonly content: RevisionContent }; + +/** + * The engine's writes as the outbox and archives record them. A creation carries its base in a commit (stored at + * `createdAt`); in an archive it has none and its `document.revision`s follow it. + */ export type DocumentWrite = - | { readonly kind: "document.create"; readonly record: DocumentRecord; readonly base: DocumentBase } + | { readonly kind: "document.create"; readonly record: DocumentRecord; readonly base?: DocumentBase } | { readonly kind: "document.change"; readonly id: number; readonly revision: number; readonly content: DocumentContent; } - | { readonly kind: "document.retire"; readonly id: number; readonly revision: number }; + | { readonly kind: "document.retire"; readonly id: number; readonly revision: number } + | ({ readonly kind: "document.revision" } & DocumentRevision); -/** A loaded document: its record and its value (base plus deltas applied once). */ +/** A loaded document: its record and its value (a base plus deltas applied once) at a point. */ export type StoredDocument = { readonly record: DocumentRecord; readonly value: JsonValue; - /** The deltas stored after its base (what `checkpointWhen` sees). */ + /** The version of the base the value was built from. */ + readonly version: number; + /** The deltas applied after its base (what `checkpointWhen` sees). */ readonly deltasSinceBase: number; }; +/** Where a read looks: the current state, or the state after a seq. */ +export type DocumentPoint = number | "current"; + /** - * Stages the creation of a document at `address` with `base`; its id comes from the step. `text` is the base's value - * serialized before the queue (rule 8). + * Stages the creation of a Chord document at `address` with `base`; its id is taken from the allocator when it + * applies (a saturated allocator refuses the commit) and handed to `created` (its owner adopts it after `COMMIT`). + * `text` is the base's value serialized before the queue (rule 8). */ export function createDocument( address: DocumentAddress, - history: "latest" | "rewindable", base: DocumentBase, text: string, + created?: (id: number) => void, ): StagedWrite { return { order: "create", apply: async (step) => { - // The next id is stored too: it stays in the safe range, or ids would stop counting exactly and repeat. - if (step.nextDocumentId >= Number.MAX_SAFE_INTEGER) { - throw new Error("store: no document id past Number.MAX_SAFE_INTEGER"); - } - const id = step.nextDocumentId++; + const id = step.ids.mint(); step.created.push(id); + created?.(id); const { version, fingerprint } = base; const record: DocumentRecord = { id, ...address, - history, + history: "none", createdAt: step.seq, revision: 1, version, fingerprint, }; await insertDocument(step, record, text); - return { kind: "document.create", record, base }; + return [{ kind: "document.create", record, base }]; }, }; } /** - * Inserts `record` as it is (id, `createdAt`, revision kept: a creation, or an import) with its base, `text` being - * the base's value serialized. + * Inserts `record` as it is (id, seqs and revision kept: a creation, an import, a batch), with its base at `createdAt` + * when `text` (the base's value serialized) is given; a record created retired (an empty lifetime) keeps what its + * history keeps. */ -export async function insertDocument(step: Step, record: DocumentRecord, text: string): Promise { - const { id, namespace, owner, kind, key, history, createdAt, meta, revision, version, fingerprint } = record; +export async function insertDocument(step: Step, record: DocumentRecord, text: string | undefined): Promise { + const { id, namespace, owner, kind, key, history, createdAt, retiredAt, meta, revision, version, fingerprint } = + record; await step.run( `INSERT INTO ${TABLES.documents} - (id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (id, owner, namespace, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, id, - namespace, owner, + namespace, kind, key ?? "", history, createdAt, + retiredAt ?? null, meta ?? null, revision, version, fingerprint, ); - await insertRevision(step, id, "base", version, text); + if (text !== undefined) await insertRevision(step, id, createdAt, "base", version, text); + if (retiredAt !== undefined) await dropRetired(step, id, history); } /** - * Stages a change of document `id` prepared on `revision`: a delta on the current base's version, or a new base - * (which prunes the older revisions of a `latest` record). `text` is the content serialized before the queue. + * Stages a change of Chord document `id` prepared on `revision`: a delta on the current base's version, or a new base. + * `text` is the content serialized before the queue. */ export function changeDocument(id: number, revision: number, content: DocumentContent, text: string): StagedWrite { const write: DocumentWrite = { kind: "document.change", id, revision, content }; return { order: "write", apply: async (step) => { - // As ids: a revision past the safe range would stop counting exactly. - if (revision >= Number.MAX_SAFE_INTEGER) throw new Error(`document ${id}: no revision past the safe range`); - await insertRevision(step, id, content.kind, content.version, text); - const changed = - content.kind === "delta" - ? await step.runCounted( - `UPDATE ${TABLES.documents} SET revision = revision + 1 - WHERE id = ? AND revision = ? AND version = ? AND retired_at IS NULL`, - id, - revision, - content.version, - ) - : await step.runCounted( - `UPDATE ${TABLES.documents} SET revision = revision + 1, version = ?, fingerprint = ? - WHERE id = ? AND revision = ? AND retired_at IS NULL`, - content.version, - content.fingerprint, - id, - revision, - ); - // Memory and disk disagree: the step fails, which stops the store (Durable's rule). - if (changed !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); - if (content.kind === "base") { - await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ? AND seq < ?`, id, step.seq); - } - return write; + await writeContent(step, id, "none", revision, content, text); + return [write]; }, }; } -/** Stages the retirement of `latest` document `id` at `revision`: its record and revisions go (rule 19). */ +/** + * Writes `content` to document `id` (of `history`) prepared on `revision`, in the step's seq: its revision, the + * counted conditional `UPDATE` (a delta only on the base's version), and at a new base the older revisions a + * `none` or `latest` record drops. A record that is not live at `revision` throws (memory and disk disagree). + */ +export async function writeContent( + step: Step, + id: number, + history: DocumentHistory, + revision: number, + content: DocumentContent, + text: string, +): Promise { + // As ids: a revision past the safe range would stop counting exactly. + if (revision >= Number.MAX_SAFE_INTEGER) throw new Error(`document ${id}: no revision past the safe range`); + await insertRevision(step, id, step.seq, content.kind, content.version, text); + const changed = + content.kind === "delta" + ? await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1 + WHERE id = ? AND revision = ? AND version = ? AND retired_at IS NULL`, + id, + revision, + content.version, + ) + : await step.runCounted( + `UPDATE ${TABLES.documents} SET revision = revision + 1, version = ?, fingerprint = ? + WHERE id = ? AND revision = ? AND retired_at IS NULL`, + content.version, + content.fingerprint, + id, + revision, + ); + if (changed !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); + if (content.kind === "base" && history !== "rewindable") { + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ? AND seq < ?`, id, step.seq); + } +} + +/** Stages the retirement of Chord document `id` (history `none`) at `revision`: its row and its revisions go. */ export function retireDocument(id: number, revision: number): StagedWrite { const write: DocumentWrite = { kind: "document.retire", id, revision }; return { order: "retire", apply: async (step) => { + // A Chord document keeps no history (`none`): its row goes with a conditional, counted DELETE (6a's). const deleted = await step.runCounted( `DELETE FROM ${TABLES.documents} WHERE id = ? AND revision = ? AND retired_at IS NULL`, id, @@ -173,43 +219,141 @@ export function retireDocument(id: number, revision: number): StagedWrite { ); if (deleted !== 1) throw new Error(`document ${id}: not live at revision ${revision}`); await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ?`, id); - return write; + return [write]; }, }; } +/** + * Retires live `record` in the step's seq by its history (R1): a `none` record's row and revisions go, a `latest` one + * keeps its row with `retired_at` and drops its revisions, a `rewindable` one keeps everything. + */ +export async function retireRecord(step: Step, record: DocumentRecord): Promise { + const { id, history } = record; + if (history !== "none") await step.run(`UPDATE ${TABLES.documents} SET retired_at = ? WHERE id = ?`, step.seq, id); + await dropRetired(step, id, history); +} + +/** What a retired record of `history` drops: a `none` row too, every revision unless `rewindable`. */ +async function dropRetired(step: Step, id: number, history: DocumentHistory): Promise { + if (history === "rewindable") return; + if (history === "none") await step.run(`DELETE FROM ${TABLES.documents} WHERE id = ?`, id); + await step.run(`DELETE FROM ${TABLES.revisions} WHERE document_id = ?`, id); +} + +/** Inserts one revision of document `id` at `seq`, `text` its serialized content. */ +export function insertRevision( + step: Step, + id: number, + seq: number, + kind: "base" | "delta", + version: number, + text: string, +): Promise { + return step.run( + `INSERT INTO ${TABLES.revisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`, + id, + seq, + kind, + version, + text, + ); +} + type RecordRow = { readonly id: number; - readonly namespace: string; readonly owner: string; + readonly namespace: string; readonly kind: string; readonly key: string; - readonly history: "latest" | "rewindable"; + readonly history: DocumentHistory; readonly created_at: number; + readonly retired_at: number | null; readonly meta: string | null; readonly revision: number; readonly version: number; readonly fingerprint: string; }; -const RECORD_COLUMNS = "id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint"; +const RECORD_COLUMNS = + "id, owner, namespace, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint"; -/** The live record at `address`, if any. */ -export async function findDocument( - executor: SqliteExecutor, - address: DocumentAddress, +const READ = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id = ?`; + +/** The record with `id`, live or retired, if there is one. */ +export async function readDocument( + executor: Pick, + id: number, ): Promise { - const row = await executor.get( - `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} - WHERE namespace = ? AND owner = ? AND kind = ? AND key = ? AND retired_at IS NULL`, - address.namespace, - address.owner, - address.kind, - address.key ?? "", - ); + const row = await executor.get(READ, id); return row === undefined ? undefined : recordOf(row); } +const FIND_CURRENT = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND retired_at IS NULL`; + +const FIND_AT = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) + ORDER BY created_at DESC LIMIT 1`; + +/** The record at `address` live at `at` (the half-open lifetime `createdAt <= at < retiredAt`), if any. */ +export async function findDocument( + executor: Pick, + address: DocumentAddress, + at: DocumentPoint = "current", +): Promise { + const { owner, namespace, kind } = address; + const key = address.key ?? ""; + const row = + at === "current" + ? await executor.get(FIND_CURRENT, owner, namespace, kind, key) + : await executor.get(FIND_AT, owner, namespace, kind, key, at, at); + return row === undefined ? undefined : recordOf(row); +} + +/** + * A record as an adapter that keeps its own fields in `meta` reads it: the meta, and, where a read can find retired + * records (at a point, by id), when it was retired. One or two columns: each column a read returns costs. + */ +export type DocumentEntry = { readonly meta: string | null; readonly retired_at?: number | null }; + +/** A document entry with what tells whose it is and what it keeps, as a read by id needs. */ +export type OwnedDocumentEntry = DocumentEntry & { + readonly owner: string; + readonly namespace: string; + readonly history: DocumentHistory; +}; + +const READ_ENTRY = `SELECT meta, retired_at, owner, namespace, history FROM ${TABLES.documents} WHERE id = ?`; + +/** The entry of the record with `id`, live or retired, with its owner, namespace and history; `undefined` if none. */ +export function readDocumentEntry( + executor: Pick, + id: number, +): Promise { + return executor.get(READ_ENTRY, id); +} + +const FIND_ENTRY_CURRENT = `SELECT meta FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND retired_at IS NULL`; + +const FIND_ENTRY_AT = `SELECT meta, retired_at FROM ${TABLES.documents} + WHERE owner = ? AND namespace = ? AND kind = ? AND key = ? AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) + ORDER BY created_at DESC LIMIT 1`; + +/** `findDocument` reading only what an adapter keeping its fields in `meta` needs (fewer columns, no record built). */ +export function findDocumentEntry( + executor: Pick, + address: DocumentAddress, + at: DocumentPoint, +): Promise { + const { owner, namespace, kind } = address; + const key = address.key ?? ""; + return at === "current" + ? executor.get(FIND_ENTRY_CURRENT, owner, namespace, kind, key) + : executor.get(FIND_ENTRY_AT, owner, namespace, kind, key, at, at); +} + type RevisionRow = { readonly seq: number; readonly kind: "base" | "delta"; @@ -217,22 +361,37 @@ type RevisionRow = { readonly content: string; }; -/** Loads the live document `record` found: its latest base and the deltas after it, applied once. */ -export async function loadDocument(executor: SqliteExecutor, record: DocumentRecord): Promise { - const revisions = await executor.all( - `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq >= ( - SELECT max(seq) FROM ${TABLES.revisions} WHERE document_id = ? AND kind = 'base') ORDER BY seq`, - record.id, - record.id, - ); +const LOAD = `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq <= ? AND seq >= ( + SELECT max(seq) FROM ${TABLES.revisions} WHERE document_id = ? AND kind = 'base' AND seq <= ?) ORDER BY seq`; + +/** + * Loads `record` at `at`: its newest base at or below it and the deltas after that base up to it, applied once; the + * version is the base's. A record without such a base (one that keeps no history from there) throws. + */ +export async function loadDocument( + executor: Pick, + record: DocumentRecord, + at: DocumentPoint = "current", +): Promise { + return { record, ...(await loadContent(executor, record.id, at)) }; +} + +/** `loadDocument`'s content of document `id` at `at`, without its record (an adapter reads the fields it keeps). */ +export async function loadContent( + executor: Pick, + id: number, + at: DocumentPoint, +): Promise> { + const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at; + const revisions = await executor.all(LOAD, id, upper, id, upper); const [base, ...deltas] = revisions; - if (base === undefined || base.kind !== "base") throw new Error(`document ${record.id}: no base`); + if (base === undefined || base.kind !== "base") throw new Error(`document ${id}: no base`); let value = JSON.parse(base.content) as JsonValue; for (const delta of deltas) { - if (delta.version !== base.version) throw new Error(`document ${record.id}: a delta crosses a version`); + if (delta.version !== base.version) throw new Error(`document ${id}: a delta crosses a version`); value = apply(value, JSON.parse(delta.content) as Op[]); } - return { record, value, deltasSinceBase: deltas.length }; + return { value, version: base.version, deltasSinceBase: deltas.length }; } /** Up to `limit` keys of live documents of `kind` under `owner` after `after` (from the first), in code point order. */ @@ -245,10 +404,10 @@ export async function listDocumentKeys( // The first page has no lower bound: `""` is a key too. const from = after === undefined ? "" : " AND key > ?"; const rows = await executor.all<{ readonly key: string }>( - `SELECT key FROM ${TABLES.documents} WHERE namespace = ? AND owner = ? AND kind = ?${from} + `SELECT key FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND kind = ?${from} AND retired_at IS NULL ORDER BY key LIMIT ?`, - address.namespace, address.owner, + address.namespace, address.kind, ...(after === undefined ? [] : [after]), limit, @@ -256,40 +415,131 @@ export async function listDocumentKeys( return rows.map(({ key }) => key); } -/** Up to `limit` live records with ids above `after`, by id (an export's pages). */ -export async function listDocuments(executor: SqliteExecutor, after: number, limit: number): Promise { +/** What `listDocuments` scans: one owner's namespace, optionally one kind. */ +export type DocumentScope = { readonly owner: string; readonly namespace: string; readonly kind?: string }; + +/** The statements of `listDocuments`, by its shape: with a kind or not, current or at a point. */ +const LIST = (() => { + const from = `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND id > ?`; + const current = " AND retired_at IS NULL ORDER BY id LIMIT ?"; + const at = " AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) ORDER BY id LIMIT ?"; + return { + current: `${from}${current}`, + at: `${from}${at}`, + kindCurrent: `${from} AND kind = ?${current}`, + kindAt: `${from} AND kind = ?${at}`, + }; +})(); + +/** Up to `limit` records of `scope` live at `at` with ids above `after`, by id. */ +export async function listDocuments( + executor: Pick, + scope: DocumentScope, + at: DocumentPoint, + after: number, + limit: number, +): Promise { + const { owner, namespace, kind } = scope; + let rows: RecordRow[]; + if (kind === undefined) { + rows = + at === "current" + ? await executor.all(LIST.current, owner, namespace, after, limit) + : await executor.all(LIST.at, owner, namespace, after, at, at, limit); + } else { + rows = + at === "current" + ? await executor.all(LIST.kindCurrent, owner, namespace, after, kind, limit) + : await executor.all(LIST.kindAt, owner, namespace, after, kind, at, at, limit); + } + return rows.map(recordOf); +} + +/** The statements of `listDocumentEntries`, by shape, as `LIST`'s. */ +const LIST_ENTRIES = (() => { + const from = (columns: string) => + `SELECT ${columns} FROM ${TABLES.documents} WHERE owner = ? AND namespace = ? AND id > ?`; + const current = " AND retired_at IS NULL ORDER BY id LIMIT ?"; + const at = " AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?) ORDER BY id LIMIT ?"; + return { + current: `${from("meta")}${current}`, + at: `${from("meta, retired_at")}${at}`, + kindCurrent: `${from("meta")} AND kind = ?${current}`, + kindAt: `${from("meta, retired_at")} AND kind = ?${at}`, + }; +})(); + +/** `listDocuments` reading only what an adapter keeping its fields in `meta` needs. */ +export function listDocumentEntries( + executor: Pick, + scope: DocumentScope, + at: DocumentPoint, + after: number, + limit: number, +): Promise { + const { owner, namespace, kind } = scope; + if (kind === undefined) { + return at === "current" + ? executor.all(LIST_ENTRIES.current, owner, namespace, after, limit) + : executor.all(LIST_ENTRIES.at, owner, namespace, after, at, at, limit); + } + return at === "current" + ? executor.all(LIST_ENTRIES.kindCurrent, owner, namespace, after, kind, limit) + : executor.all(LIST_ENTRIES.kindAt, owner, namespace, after, kind, at, at, limit); +} + +/** Up to `limit` stored records, live and retired, with ids above `after`, by id (an export's pages). */ +export async function listRecords(executor: SqliteExecutor, after: number, limit: number): Promise { const rows = await executor.all( - `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id > ? AND retired_at IS NULL ORDER BY id LIMIT ?`, + `SELECT ${RECORD_COLUMNS} FROM ${TABLES.documents} WHERE id > ? ORDER BY id LIMIT ?`, after, limit, ); return rows.map(recordOf); } +/** Up to `limit` stored revisions of document `id` after seq `after`, in seq order (an export's page). */ +export async function listRevisions( + executor: SqliteExecutor, + id: number, + after: number, + limit: number, +): Promise { + const rows = await executor.all( + `SELECT seq, kind, version, content FROM ${TABLES.revisions} WHERE document_id = ? AND seq > ? ORDER BY seq LIMIT ?`, + id, + after, + limit, + ); + return rows.map(({ seq, kind, version, content }) => { + // Written by `insertRevision`: a base's value or a delta's ops, as JSON. + const parsed = JSON.parse(content) as unknown; + return { + id, + seq, + content: + kind === "base" + ? { kind, version, value: parsed as JsonValue } + : { kind, version, ops: parsed as readonly Op[] }, + }; + }); +} + function recordOf(row: RecordRow): DocumentRecord { - const { id, namespace, owner, kind, key, history, created_at, meta, revision, version, fingerprint } = row; + const { id, namespace, owner, kind, key, history, created_at, retired_at, meta, revision, version, fingerprint } = + row; return { id, - namespace, owner, + namespace, kind, ...(key === "" ? {} : { key }), history, createdAt: created_at, + ...(retired_at === null ? {} : { retiredAt: retired_at }), ...(meta === null ? {} : { meta }), revision, version, fingerprint, }; } - -function insertRevision(step: Step, id: number, kind: "base" | "delta", version: number, text: string): Promise { - return step.run( - `INSERT INTO ${TABLES.revisions} (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)`, - id, - step.seq, - kind, - version, - text, - ); -} diff --git a/packages/chord/src/next/store/export.ts b/packages/chord/src/next/store/export.ts index cdadc0424..78554d6e2 100644 --- a/packages/chord/src/next/store/export.ts +++ b/packages/chord/src/next/store/export.ts @@ -10,7 +10,7 @@ import type { SqliteValue } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; import { ARCHIVE_FORMAT, type ArchivePart } from "./archive.ts"; import type { TableRecord } from "./catalog.ts"; -import { listDocuments, loadDocument } from "./engine.ts"; +import { listRecords, listRevisions } from "./engine.ts"; import { ROW_VERSION, type Scalar, versionColumn } from "./rows.ts"; import { STORE_SCHEMA } from "./schema.ts"; import { quote, recordedPut } from "./sql.ts"; @@ -116,8 +116,8 @@ async function* parts( hold(release); try { if (context.abortSignal?.aborted) throw aborted(context); - const { lastSeq, nextDocumentId } = store; - yield { kind: "manifest", format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq, nextDocumentId }; + const { lastSeq, nextId } = store; + yield { kind: "manifest", format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq, nextId }; let writes = 0; const tables = [...store.tables.values()].sort((a, b) => (a.table < b.table ? -1 : 1)); for (const record of tables) { @@ -133,8 +133,8 @@ async function* parts( writes++; } } - for await (const create of documents(store, context)) { - yield create; + for await (const part of documents(store, context)) { + yield part; writes++; } yield { kind: "end", writes }; @@ -188,20 +188,37 @@ function putOf( return { kind: "put", table: record.table, row, version: stored[ROW_VERSION] as number, columns: versions }; } -/** The live documents by id as their creations: the record and the current value as the base. */ +/** + * Every stored record by id, live and retired, as its creation without a base, then its kept revisions in seq order: + * a `none` or `latest` record's current base and its deltas, a retired `latest` one none, a `rewindable` one all. + */ async function* documents( store: ChordStore, context: Context, -): AsyncGenerator> { +): AsyncGenerator> { let after = 0; for (;;) { - const records = await store.read((executor) => listDocuments(executor, after, EXPORT_PAGE), context); + const records = await store.read((executor) => listRecords(executor, after, EXPORT_PAGE), context); for (const record of records) { - const { value } = await store.read((executor) => loadDocument(executor, record), context); - const { version, fingerprint } = record; - yield { kind: "document.create", record, base: { version, fingerprint, value } }; + yield { kind: "document.create", record }; + yield* revisions(store, record.id, context); } if (records.length < EXPORT_PAGE) return; after = records.at(-1)!.id; } } + +/** Document `id`'s kept revisions in seq order, read page by page (a long history is never one array). */ +async function* revisions( + store: ChordStore, + id: number, + context: Context, +): AsyncGenerator> { + let after = 0; + for (;;) { + const page = await store.read((executor) => listRevisions(executor, id, after, EXPORT_PAGE), context); + for (const revision of page) yield { kind: "document.revision", ...revision }; + if (page.length < EXPORT_PAGE) return; + after = page.at(-1)!.seq; + } +} diff --git a/packages/chord/src/next/store/import.ts b/packages/chord/src/next/store/import.ts index c53ca4e77..46d4d290d 100644 --- a/packages/chord/src/next/store/import.ts +++ b/packages/chord/src/next/store/import.ts @@ -1,17 +1,20 @@ // `importStore(backend, parts, ctx, { outbox? })` (slice 6a §13, rules 12, 24, 25): a store made from an archive, a // trust boundary. Every part is checked against the archive schema, its place in the archive's order and its content -// (names, rows against their table's recorded columns, ids and seqs, one live record per address, versions, -// duplicates, the `end` count, the manifest); everything is applied in one backend transaction with the applier, so a -// refused import leaves the backend as it was. An input that stalls is cancelled by `ctx`. +// (names, rows against their table's recorded columns, ids and seqs, lifetimes at one address never overlapping, +// revisions in seq order from a base, versions, duplicates, the `end` count, the manifest); everything is applied in +// one backend transaction with the applier, so a refused import leaves the backend as it was. An input that stalls is +// cancelled by `ctx`. import { awaitWithContext, type Context } from "../../context/index.ts"; import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; import { invalid } from "../data/internal.ts"; +import { IdAllocator } from "./allocator.ts"; import { applyWrite } from "./applier.ts"; import { ARCHIVE_FORMAT, type ArchiveManifest, type ArchivePart, checkPart } from "./archive.ts"; import type { Backend } from "./backend.ts"; import { readCatalog, writeCatalog } from "./catalog.ts"; +import type { DocumentRecord } from "./engine.ts"; import { applyStoreSchema, type MetaRow, STORE_SCHEMA, TABLES } from "./schema.ts"; import { Step } from "./step.ts"; import { ChordStore, type Store, type StoreOptions } from "./store.ts"; @@ -83,7 +86,10 @@ class ArchiveImport { #table: string | undefined; readonly #tables = new Set(); readonly #ids = new Set(); - readonly #addresses = new Set(); + /** The lifetimes seen at each address, `[createdAt, retiredAt)` (`Infinity` while live). */ + readonly #lifetimes = new Map(); + /** The record whose revisions may follow, with the last revision's seq and its base's version. */ + #document: { readonly record: DocumentRecord; seq: number; base: number | undefined } | undefined; #writes = 0; constructor(executor: SqliteExecutor, context: Context) { @@ -105,6 +111,7 @@ class ArchiveImport { if (part.writes !== this.#writes) { throw invalid("args", `${what}: counts ${part.writes} writes, the archive has ${this.#writes}`); } + this.#endDocument(what); this.#place = "ended"; return; } @@ -113,24 +120,24 @@ class ArchiveImport { await applyWrite(this.#step, part, what); } - /** The stored metadata once the archive ended: its `lastSeq`, next document id and the outbox setting. */ + /** The stored metadata once the archive ended: its `lastSeq`, the allocator's next id and the outbox setting. */ async finish(meta: MetaRow, outbox: boolean): Promise { if (this.#place !== "ended" || this.#step === undefined || this.#manifest === undefined) { throw invalid("args", "import: the archive has no end part"); } - const { lastSeq, nextDocumentId } = this.#manifest; + const { lastSeq, nextId } = this.#manifest; await writeCatalog(this.#step); await this.#executor.run( - `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ?, outbox = ?, outbox_floor = ? WHERE singleton = 1`, + `UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ?, outbox = ?, outbox_floor = ? WHERE singleton = 1`, lastSeq, - nextDocumentId, + nextId, outbox ? 1 : 0, lastSeq, ); return { ...meta, last_seq: lastSeq, - next_document_id: nextDocumentId, + next_id: nextId, outbox: outbox ? 1 : 0, outbox_floor: lastSeq, }; @@ -145,7 +152,7 @@ class ArchiveImport { if (part.schema > STORE_SCHEMA) throw invalid("args", `${what}: schema ${part.schema} is newer than ${STORE_SCHEMA}`); this.#manifest = part; - this.#step = new Step(this.#executor, part.lastSeq, part.nextDocumentId, new Map()); + this.#step = new Step(this.#executor, part.lastSeq, new IdAllocator(part.nextId), new Map()); } /** Where `part` puts the archive, refused when out of order, a duplicate, or outside the manifest's ranges. */ @@ -171,21 +178,27 @@ class ArchiveImport { case "document.create": this.#checkDocument(part, what); return "documents"; + case "document.revision": + this.#checkRevision(part, what); + return "documents"; } } /** - * A record within the manifest's ids and seqs, unique, alone at its address, its base at its version; its address - * and meta well-formed text without NUL (SQLite would cut a NUL off, slice 6a §8). + * A record within the manifest's ids and seqs (an id equal to a saturated `nextId` too), unique, its lifetime + * overlapping no other at its address; its address and meta well-formed text without NUL (SQLite would cut a NUL off, + * slice 6a §8). The record before it must have ended well (`#endDocument`). */ #checkDocument(part: Extract, what: string): void { - const { record, base } = part; - const { lastSeq, nextDocumentId } = this.#manifest!; - if (record.id >= nextDocumentId) throw invalid("args", `${what}: id ${record.id} is not below ${nextDocumentId}`); + this.#endDocument(what); + const { record } = part; + const { lastSeq, nextId } = this.#manifest!; + const saturated = nextId === Number.MAX_SAFE_INTEGER && record.id === nextId; + if (record.id >= nextId && !saturated) throw invalid("args", `${what}: id ${record.id} is not below ${nextId}`); if (this.#ids.has(record.id)) throw invalid("args", `${what}: id ${record.id} occurs twice`); - if (record.createdAt > lastSeq) throw invalid("args", `${what}: created after the manifest's lastSeq`); - if (base.version !== record.version || base.fingerprint !== record.fingerprint) { - throw invalid("args", `${what}: the base is not at the record's version and fingerprint`); + const { createdAt, retiredAt } = record; + if (createdAt > lastSeq || (retiredAt !== undefined && (retiredAt < createdAt || retiredAt > lastSeq))) { + throw invalid("args", `${what}: a lifetime outside the manifest's lastSeq`); } const { namespace, owner, kind, meta } = record; // A singleton's record has no key; its column keeps `""`. @@ -196,10 +209,60 @@ class ArchiveImport { if (meta !== undefined && storedText(meta, what) !== meta) { throw invalid("args", `${what}: the meta is not well-formed text`); } - const address = JSON.stringify([namespace, owner, kind, key]); - if (this.#addresses.has(address)) throw invalid("args", `${what}: a second live record at one address`); + const address = JSON.stringify([owner, namespace, kind, key]); + const lifetime: [number, number] = [createdAt, retiredAt ?? Number.POSITIVE_INFINITY]; + const lifetimes = this.#lifetimes.get(address) ?? []; + // Two live records are two infinite lifetimes, which overlap; an empty one (`createdAt = retiredAt`) overlaps none. + if (lifetime[0] < lifetime[1] && lifetimes.some(([from, to]) => from < lifetime[1] && lifetime[0] < to)) { + throw invalid("args", `${what}: a second record at one address for an overlapping lifetime`); + } + lifetimes.push(lifetime); + this.#lifetimes.set(address, lifetimes); this.#ids.add(record.id); - this.#addresses.add(address); + this.#document = { record, seq: 0, base: undefined }; + } + + /** + * A revision of the record before it, in seq order within its lifetime, from a base, each delta at its base's + * version. + */ + #checkRevision(part: Extract, what: string): void { + const document = this.#document; + if (document === undefined || document.record.id !== part.id) { + throw invalid("args", `${what}: a revision follows its record's create`); + } + const { record } = document; + if (record.retiredAt !== undefined && record.history !== "rewindable") { + throw invalid("args", `${what}: a retired ${record.history} record keeps no revisions`); + } + if ( + part.seq <= document.seq || + part.seq < record.createdAt || + part.seq > (record.retiredAt ?? this.#manifest!.lastSeq) + ) { + throw invalid("args", `${what}: a revision out of seq order or outside its lifetime`); + } + const { content } = part; + if (content.kind === "base") document.base = content.version; + else if (document.base !== content.version) { + throw invalid("args", `${what}: a delta without a base at its version`); + } + document.seq = part.seq; + } + + /** + * The record before ends well: its last base is at the record's version; only a retired `latest` record has none + * (its content was dropped at retirement; a retired `rewindable` one keeps its history). + */ + #endDocument(what: string): void { + const document = this.#document; + this.#document = undefined; + if (document === undefined) return; + const { record, base } = document; + const contentless = record.retiredAt !== undefined && record.history === "latest"; + if (base === undefined ? !contentless : base !== record.version) { + throw invalid("args", `${what}: record ${record.id} has no base at its version`); + } } } diff --git a/packages/chord/src/next/store/outbox.ts b/packages/chord/src/next/store/outbox.ts index 642feba73..c9b1622e3 100644 --- a/packages/chord/src/next/store/outbox.ts +++ b/packages/chord/src/next/store/outbox.ts @@ -11,8 +11,8 @@ import type { StoreWrite } from "./writes.ts"; /** The write vocabulary's version an outbox commit is stored in; a newer one is refused, data intact. */ export const OUTBOX_FORMAT = 1; -/** One commit as the outbox gives it. */ -export type OutboxCommit = { readonly seq: number; readonly writes: readonly StoreWrite[] }; +/** One commit as the outbox gives it: its seq, the allocator's value it persisted (D3) and its writes in applied order. */ +export type OutboxCommit = { readonly seq: number; readonly nextId: number; readonly writes: readonly StoreWrite[] }; const NOTHING = (): void => {}; @@ -73,19 +73,24 @@ export class Outbox { } const rows = await this.#store.read( (executor) => - executor.all<{ readonly seq: number; readonly format: number; readonly writes: string }>( - `SELECT seq, format, writes FROM ${TABLES.outbox} WHERE seq > ? ORDER BY seq LIMIT ?`, + executor.all<{ + readonly seq: number; + readonly format: number; + readonly next_id: number; + readonly writes: string; + }>( + `SELECT seq, format, next_id, writes FROM ${TABLES.outbox} WHERE seq > ? ORDER BY seq LIMIT ?`, afterSeq, limit, ), context, ); - return rows.map(({ seq, format, writes }) => { + return rows.map(({ seq, format, next_id, writes }) => { if (format !== OUTBOX_FORMAT) { const message = `outbox: commit ${seq} is in format ${format}, newer than ${OUTBOX_FORMAT}`; throw new ChordError({ code: "unavailable", message }); } - return { seq, writes: JSON.parse(writes) as StoreWrite[] }; + return { seq, nextId: next_id, writes: JSON.parse(writes) as StoreWrite[] }; }); } diff --git a/packages/chord/src/next/store/refusal.ts b/packages/chord/src/next/store/refusal.ts new file mode 100644 index 000000000..2ccb3ce52 --- /dev/null +++ b/packages/chord/src/next/store/refusal.ts @@ -0,0 +1,43 @@ +// A refused commit as Chord's owners take it (slice 6b D4, §1.4): the store rejects a step it rolled back with the +// step's own error, unchanged (Durable's Storage semantics); Chord classifies it: an `int` column's `CHECK` (a +// plugin's value out of range) fails only its transaction, any other refusal means memory and disk disagree. + +import { ChordError, isChordError } from "../data/index.ts"; + +/** How a Chord commit ended that the store rejected: `check` fails the transaction only, `broken` its object too. */ +export type Refusal = + | { readonly kind: "check"; readonly error: ChordError } + | { readonly kind: "broken"; readonly error: ChordError }; + +/** SQLite's extended result code `SQLITE_CONSTRAINT_CHECK`, which Node's binding reports as `errcode`. */ +const SQLITE_CONSTRAINT_CHECK = 275; + +/** + * The refusal behind `error`, a store commit's rejection; `undefined` when it is the store's own (`released` at + * admission or after close, `unavailable/storage_failed` once stopped), which the owner rethrows as it is. + */ +export function refusalOf(error: unknown, label: string): Refusal | undefined { + if (isChordError(error, "released") || (isChordError(error, "unavailable") && error.reason === "storage_failed")) { + return undefined; + } + const message = error instanceof Error ? error.message : String(error); + if (isCheck(error, message)) { + const refused = `${label}: a value is out of its column's range (${message})`; + return { + kind: "check", + error: new ChordError({ code: "invalid", reason: "args", message: refused }, { cause: error }), + }; + } + const refused = `${label}: the store refused the commit (${message})`; + return { kind: "broken", error: new ChordError({ code: "unavailable", message: refused }, { cause: error }) }; +} + +/** + * Whether a refusal is a `CHECK` failure: SQLite's extended code where the binding has one (Node), the message + * otherwise (the Durable Object facade has no extended codes). + */ +function isCheck(error: unknown, message: string): boolean { + const code = (error as { readonly errcode?: unknown } | undefined)?.errcode; + if (typeof code === "number") return code === SQLITE_CONSTRAINT_CHECK; + return message.includes("CHECK constraint failed"); +} diff --git a/packages/chord/src/next/store/schema.ts b/packages/chord/src/next/store/schema.ts index c528360ce..249d177f7 100644 --- a/packages/chord/src/next/store/schema.ts +++ b/packages/chord/src/next/store/schema.ts @@ -1,6 +1,6 @@ // The store's own SQL schema (slice 6a §2): its metadata, the recorded table definitions, the documents engine's // records and revisions, and the outbox; versioned by `applySqliteMigrations`. Declared tables are named -// `"::
"`, so no name here can meet one of them. +// `":
"`, so no name here can meet one of them. import { applySqliteMigrations, type SqliteDatabase, type SqliteMigration } from "../../sqlite/index.ts"; @@ -14,10 +14,13 @@ export const TABLES = { } as const; /** - * The schema history; never change an entry, append one. `chord_meta` is one row: `last_seq`, the next document id - * (S10), and the outbox setting with its floor (the seq it starts after, advanced by `trim`). A document record keeps - * its current base's version and fingerprint and a `revision` counter for the conditional `UPDATE`; the partial - * unique index refuses two live records at one address. + * The schema history; never change an entry, append one (nothing is deployed yet: slice 6b changed entry 1 in place). + * `chord_meta` is one row: `last_seq`, the allocator's next id (D3; from 2, id 1 is Durable's root conversation), and + * the outbox setting with its floor (the seq it starts after, advanced by `trim`). A document record keeps its + * history rule (R1), its current base's version and fingerprint and a `revision` counter for the conditional + * `UPDATE`; the partial unique index refuses two live records at one address. Entry 2 adds the indexes of reads at a + * point (slice 6b §1.5): an address's incarnations by creation, an owner's namespace by id, a document's bases (a + * partial index: deltas write no entry). */ const MIGRATIONS: readonly SqliteMigration[] = [ { @@ -26,19 +29,19 @@ const MIGRATIONS: readonly SqliteMigration[] = [ `CREATE TABLE ${TABLES.meta} ( singleton INTEGER PRIMARY KEY CHECK (singleton = 1), last_seq INTEGER NOT NULL, - next_document_id INTEGER NOT NULL, + next_id INTEGER NOT NULL, outbox INTEGER NOT NULL CHECK (outbox IN (0, 1)), outbox_floor INTEGER NOT NULL ) STRICT`, - `INSERT INTO ${TABLES.meta} (singleton, last_seq, next_document_id, outbox, outbox_floor) VALUES (1, 0, 1, 0, 0)`, + `INSERT INTO ${TABLES.meta} (singleton, last_seq, next_id, outbox, outbox_floor) VALUES (1, 0, 2, 0, 0)`, `CREATE TABLE ${TABLES.tables} (name TEXT PRIMARY KEY, definition TEXT NOT NULL) STRICT, WITHOUT ROWID`, `CREATE TABLE ${TABLES.documents} ( id INTEGER PRIMARY KEY, - namespace TEXT NOT NULL, owner TEXT NOT NULL, + namespace TEXT NOT NULL, kind TEXT NOT NULL, key TEXT NOT NULL, - history TEXT NOT NULL CHECK (history IN ('latest', 'rewindable')), + history TEXT NOT NULL CHECK (history IN ('none', 'latest', 'rewindable')), created_at INTEGER NOT NULL, retired_at INTEGER, meta TEXT, @@ -46,7 +49,7 @@ const MIGRATIONS: readonly SqliteMigration[] = [ version INTEGER NOT NULL, fingerprint TEXT NOT NULL ) STRICT`, - `CREATE UNIQUE INDEX chord_documents_live ON ${TABLES.documents} (namespace, owner, kind, key) + `CREATE UNIQUE INDEX chord_documents_live ON ${TABLES.documents} (owner, namespace, kind, key) WHERE retired_at IS NULL`, `CREATE TABLE ${TABLES.revisions} ( document_id INTEGER NOT NULL, @@ -56,7 +59,21 @@ const MIGRATIONS: readonly SqliteMigration[] = [ content TEXT NOT NULL, PRIMARY KEY (document_id, seq) ) STRICT, WITHOUT ROWID`, - `CREATE TABLE ${TABLES.outbox} (seq INTEGER PRIMARY KEY, format INTEGER NOT NULL, writes TEXT NOT NULL) STRICT`, + `CREATE TABLE ${TABLES.outbox} ( + seq INTEGER PRIMARY KEY, + format INTEGER NOT NULL, + next_id INTEGER NOT NULL, + writes TEXT NOT NULL + ) STRICT`, + ], + }, + { + version: 2, + statements: [ + `CREATE INDEX chord_documents_address ON ${TABLES.documents} (owner, namespace, kind, key, created_at)`, + `CREATE INDEX chord_documents_scope ON ${TABLES.documents} (owner, namespace, id)`, + // Only bases are looked up by it (a load's newest base at or below a point): deltas write no entry. + `CREATE INDEX chord_revisions_bases ON ${TABLES.revisions} (document_id, seq) WHERE kind = 'base'`, ], }, ]; @@ -67,23 +84,44 @@ export const STORE_SCHEMA = MIGRATIONS.at(-1)!.version; /** The store's metadata row, as `openStore` reads it once. */ export type MetaRow = { readonly last_seq: number; - readonly next_document_id: number; + readonly next_id: number; readonly outbox: number; readonly outbox_floor: number; }; /** * Brings `db` to the store's newest schema (one transaction) and checks that it stores text as UTF-8 (the store - * compares text by its bytes); returns the metadata row. + * compares text by its bytes); returns the metadata row. A store already there is read in one statement, without the + * migrations' write transaction (most opens: the cost of a reopen). */ export async function applyStoreSchema(db: SqliteDatabase): Promise { + const current = await currentMeta(db); + if (current !== undefined) return current; const encoding = await db.get<{ readonly encoding: string }>("PRAGMA encoding"); if (encoding?.encoding !== "UTF-8") throw new Error(`store: the database encoding is ${encoding?.encoding}, not UTF-8`); await applySqliteMigrations(db, MIGRATIONS, { table: "chord_store_schema" }); const meta = await db.get( - `SELECT last_seq, next_document_id, outbox, outbox_floor FROM ${TABLES.meta} WHERE singleton = 1`, + `SELECT last_seq, next_id, outbox, outbox_floor FROM ${TABLES.meta} WHERE singleton = 1`, ); if (meta === undefined) throw new Error("store: the metadata row is missing"); return meta; } + +/** The schema version, the encoding and the metadata row of a store, in one read. */ +const CURRENT_META = `SELECT s.version, e.encoding, m.last_seq, m.next_id, m.outbox, m.outbox_floor + FROM chord_store_schema s, pragma_encoding e, ${TABLES.meta} m WHERE s.singleton = 1 AND m.singleton = 1`; + +/** The metadata row of a store at the newest schema in UTF-8; `undefined` for any other database. */ +async function currentMeta(db: SqliteDatabase): Promise { + let row: (MetaRow & { readonly version: number; readonly encoding: string }) | undefined; + try { + row = await db.get(CURRENT_META); + } catch { + // A database without the store's tables (new, or not a store) or a backend without `pragma_encoding`: the + // migrations' path creates, refuses or reads it. + return undefined; + } + if (row === undefined || row.version !== STORE_SCHEMA || row.encoding !== "UTF-8") return undefined; + return { last_seq: row.last_seq, next_id: row.next_id, outbox: row.outbox, outbox_floor: row.outbox_floor }; +} diff --git a/packages/chord/src/next/store/sql.ts b/packages/chord/src/next/store/sql.ts index 95529eb55..867b15bd5 100644 --- a/packages/chord/src/next/store/sql.ts +++ b/packages/chord/src/next/store/sql.ts @@ -19,13 +19,14 @@ export function quote(name: string): string { return `"${name}"`; } -/** The SQL name of an object's table: `::
`, injective since no part contains `:`. */ -export function tableSqlName(hostKind: string, objectType: string, table: string): string { - return `${hostKind}:${objectType}:${table}`; +/** The SQL name of an owner's table: `:
` (R2), injective since neither part contains `:`. */ +export function tableSqlName(owner: string, table: string): string { + return `${owner}:${table}`; } /** - * `CREATE TABLE` for `columns` (as recorded) and `key`: `STRICT`, `WITHOUT ROWID`, the key as primary key, an `int` + * `CREATE TABLE` for `columns` (as recorded) and `key`: `STRICT`, the key as primary key (`WITHOUT ROWID`, except + * for a single `int` key, which becomes the rowid: the fastest point lookups, and rows of any size), an `int` * column checked within ±(2^53 - 1) (rule 10), a `bool` one within 0 and 1, a `json(...)` one as `TEXT` with its * version column, and the row's `$v`. */ @@ -37,7 +38,8 @@ export function createTableSql( const definitions = Object.entries(columns).flatMap(([name, column]) => columnSql(name, column)); definitions.push(`${quote(ROW_VERSION)} INTEGER NOT NULL`); definitions.push(`PRIMARY KEY (${key.map(quote).join(", ")})`); - return `CREATE TABLE ${quote(table)} (${definitions.join(", ")}) STRICT, WITHOUT ROWID`; + const rowid = key.length === 1 && columns[key[0]!]!.type === "int"; + return `CREATE TABLE ${quote(table)} (${definitions.join(", ")}) STRICT${rowid ? "" : ", WITHOUT ROWID"}`; } const SAFE = Number.MAX_SAFE_INTEGER; @@ -114,7 +116,12 @@ export class TableSql { readonly get: string; readonly delete: string; - constructor(table: string, columns: readonly Column[], physical: readonly string[], key: readonly string[]) { + constructor( + table: string, + columns: readonly Pick[], + physical: readonly string[], + key: readonly string[], + ) { this.table = table; const id = quote(table); this.#select = [ @@ -127,6 +134,11 @@ export class TableSql { this.delete = `DELETE FROM ${id} WHERE ${this.#byKey}`; } + /** `SELECT` of `columns` of one row by key: a point lookup, without the `ORDER BY` and `LIMIT` of a page. */ + getColumns(columns: readonly string[]): string { + return `SELECT ${columns.map(quote).join(", ")} FROM ${quote(this.table)} WHERE ${this.#byKey}`; + } + /** `UPDATE` of one row by key; `versions` gives each `json(...)` column's version to stamp. */ update(set: SetNode, key: readonly SqliteValue[], versions: ReadonlyMap): Statement { const params: SqliteValue[] = []; @@ -151,13 +163,14 @@ export class TableSql { /** * A page: rows matching `where` in `order`, after the order values `after` (row-value comparison; the key in the - * order makes it total), at most `limit` (bound). + * order makes it total), at most `limit` (bound); every column, or only `columns`. */ select( where: WhereNode | undefined, order: Order, after: readonly SqliteValue[] | undefined, limit: number, + columns?: readonly string[], ): Statement { const params: SqliteValue[] = []; const conditions: string[] = []; @@ -171,7 +184,8 @@ export class TableSql { const direction = order.direction === "asc" ? "ASC" : "DESC"; const sorting = order.columns.map((column) => `${quote(column)} ${direction}`).join(", "); params.push(limit); - return { sql: `SELECT ${this.#select} FROM ${quote(this.table)}${filter} ORDER BY ${sorting} LIMIT ?`, params }; + const select = columns === undefined ? this.#select : columns.map(quote).join(", "); + return { sql: `SELECT ${select} FROM ${quote(this.table)}${filter} ORDER BY ${sorting} LIMIT ?`, params }; } /** `count(*)` of the rows matching `where`. */ diff --git a/packages/chord/src/next/store/step.ts b/packages/chord/src/next/store/step.ts index 8a428774a..57be88756 100644 --- a/packages/chord/src/next/store/step.ts +++ b/packages/chord/src/next/store/step.ts @@ -1,30 +1,38 @@ -// The commit step's working state (slice 6a §3) and the shape every write takes into it: a write staged before the -// queue (serialized, rule 8), applied inside the step in rule 9's order. +// The commit step's working state (slice 6a §3, slice 6b §1.3) and the shape every write takes into it: a write staged +// before the queue (serialized, rule 8), applied inside the step, returning the writes it recorded. import type { SqliteExecutor, SqliteValue } from "../../sqlite/index.ts"; +import type { IdAllocator } from "./allocator.ts"; import type { TableRecord } from "./catalog.ts"; import type { StoreWrite } from "./writes.ts"; -/** Where a staged write goes in its commit (rule 9): retirements, creations, then everything else in order. */ +/** + * Where a staged write goes in its commit: retirements, creations, then everything else, each phase in staging order. + * A retirement frees a live address before a creation takes it (the partial unique index); an adapter's composite + * write orders its own parts inside `write`. + */ export type WriteOrder = "retire" | "create" | "write"; -/** A write staged for the commit step: its order and what it does there, returning the write as recorded. */ +/** A write staged for the commit step: its order and what it does there, returning the writes as recorded. */ export type StagedWrite = { readonly order: WriteOrder; - /** Applies the write inside the step's backend transaction; resolves with the write as the outbox records it. */ - apply(step: Step): Promise; + /** + * Applies the write inside the step's backend transaction; resolves with the writes as the outbox records them, in + * the order applied, with what only the step knows resolved (its seq, a record's revision). + */ + apply(step: Step): Promise; }; /** - * One commit step: its backend transaction, its seq, the document ids it assigns (S10, monotonic with seq), the table - * records it changed (the store's catalog takes them after `COMMIT`) and the statements it ran - * (`chord.store.statements`). + * One commit step: its backend transaction, its seq, the store's id allocator (D3: ids are taken from it as the + * writes apply, never copied), the ids of the documents it created, the table records it changed (the store's catalog + * takes them after `COMMIT`) and the statements it ran (`chord.store.statements`). Its reads are the step reader of + * adapters: they see the step's own writes. */ export class Step { readonly executor: SqliteExecutor; readonly seq: number; - /** The next document id; the store adopts it after `COMMIT`. */ - nextDocumentId: number; + readonly ids: IdAllocator; /** The ids of the commit's created documents, in the order of their writes. */ readonly created: number[] = []; /** The table records this step changed, by SQL name. */ @@ -32,15 +40,10 @@ export class Step { readonly #catalog: ReadonlyMap; statements = 0; - constructor( - executor: SqliteExecutor, - seq: number, - nextDocumentId: number, - catalog: ReadonlyMap, - ) { + constructor(executor: SqliteExecutor, seq: number, ids: IdAllocator, catalog: ReadonlyMap) { this.executor = executor; this.seq = seq; - this.nextDocumentId = nextDocumentId; + this.ids = ids; this.#catalog = catalog; } @@ -61,6 +64,12 @@ export class Step { return this.executor.runCounted(sql, ...params); } + /** Runs one query for its first row. */ + get(sql: string, ...params: SqliteValue[]): Promise { + this.statements++; + return this.executor.get(sql, ...params); + } + /** Runs one query. */ all(sql: string, ...params: SqliteValue[]): Promise { this.statements++; diff --git a/packages/chord/src/next/store/store.ts b/packages/chord/src/next/store/store.ts index f24f2d51e..cde65fea2 100644 --- a/packages/chord/src/next/store/store.ts +++ b/packages/chord/src/next/store/store.ts @@ -1,13 +1,19 @@ -// The store (slice 6a §2, §3): opened over a backend, one commit queue that serializes only the commit step (the seal -// fence's admission check, `BEGIN`, the writes in rule 9's order, the outbox commit, `lastSeq`, `COMMIT`, then the -// owner's adoption), reads guarded as Durable's `GuardedStorage` guards them, and the stop after a backend error. +// The store (slice 6a §2, §3, slice 6b §1): opened over a backend, one commit queue that serializes only the commit step +// (the seal fence's admission check, `BEGIN`, the writes in their order, the outbox commit, `lastSeq` and the +// allocator's value, `COMMIT`, then the owner's adoption), the id allocator, the claims of its tables and documents, +// reads guarded as Durable's `GuardedStorage` guards them, Durable's Storage semantics for a refused step (D4), and the +// stop after an unknown outcome. import { awaitWithContext, type Context } from "../../context/index.ts"; import type { SqliteDatabase, SqliteExecutor } from "../../sqlite/index.ts"; import { ChordError } from "../data/index.ts"; +import { invalid } from "../data/internal.ts"; +import type { Batch } from "../state/internal.ts"; +import { IdAllocator } from "./allocator.ts"; import type { ArchivePart } from "./archive.ts"; import type { Backend } from "./backend.ts"; import { readCatalog, type TableRecord, writeCatalog } from "./catalog.ts"; +import { Claims } from "./claims.ts"; import { StoreExport } from "./export.ts"; import { OUTBOX_FORMAT, Outbox } from "./outbox.ts"; import { applyStoreSchema, type MetaRow, TABLES } from "./schema.ts"; @@ -81,6 +87,19 @@ export type CommitFence = { adopt(seq: number, ids: readonly number[]): void; }; +/** What the open host gives the store: its `storage_failed` listener and its batch, in which adapters adopt (§1.3). */ +export type Attachment = { + readonly failed: (error: unknown) => void; + readonly batch: Batch; +}; + +/** Where a replayed batch leaves the store: its last commit's seq and allocator value, the tables it changed. */ +export type Replayed = { + readonly lastSeq: number; + readonly nextId: number; + readonly tables: ReadonlyMap; +}; + /** Where a store stands: open, stopped by a backend error, or closed. */ type StoreState = | { readonly kind: "open" } @@ -91,19 +110,30 @@ const ORDER: readonly WriteOrder[] = ["retire", "create", "write"]; const NOTHING = (): void => {}; +/** The step's own error, until its callback throws one: a distinct value no thrown error can be. */ +const NONE: unique symbol = Symbol("none"); + /** The store behind `Store`, with what Chord's layers use: commits, guarded reads, the host's failure listener. */ export class ChordStore implements Store { readonly #db: SqliteDatabase; #lastSeq: number; - #nextDocumentId: number; + /** The allocator's value as the last commit persisted it (an export's manifest). */ + #committedNextId: number; + readonly #ids: IdAllocator; + /** The claims of its tables and documents (slice 6b). */ + readonly claims = new Claims(); #outbox: Outbox | undefined; #state: StoreState = { kind: "open" }; + /** Set once a batch's recorded count did not reproduce (S17): no batch applies any more. */ + #diverged = false; /** The commit queue's tail: each step starts after the one before it settled. */ #tail: Promise = Promise.resolve(); - /** Reads underway, which `close` waits for. */ - readonly #underway = new Set>(); - /** The open host's listener for `storage_failed`, told once; one host per store (rule 7). */ - #listener: ((error: unknown) => void) | undefined; + /** Reads and maintenance writes underway, which `close` waits for. */ + #underway = 0; + /** Resolves `close`'s wait once no read is underway. */ + #drained: (() => void) | undefined; + /** The open host's listener for `storage_failed` (told once) and batch; one host per store (rule 7). */ + #attachment: Attachment | undefined; #closing: Promise | undefined; /** The recorded tables, by SQL name; changed only after a step's `COMMIT`. */ readonly #catalog: Map; @@ -114,7 +144,8 @@ export class ChordStore implements Store { this.#db = db; this.#catalog = catalog; this.#lastSeq = meta.last_seq; - this.#nextDocumentId = meta.next_document_id; + this.#committedNextId = meta.next_id; + this.#ids = new IdAllocator(meta.next_id); this.#outbox = meta.outbox === 1 ? new Outbox(this, meta.outbox_floor) : undefined; } @@ -126,9 +157,25 @@ export class ChordStore implements Store { return this.#outbox; } - /** The id the next created document gets. */ - get nextDocumentId(): number { - return this.#nextDocumentId; + /** The allocator's value as the last commit persisted it. */ + get nextId(): number { + return this.#committedNextId; + } + + /** The id allocator: `mint` takes the next id, `raise` makes room above an explicit one (D3). */ + get ids(): IdAllocator { + return this.#ids; + } + + /** Takes the next id from the allocator (Durable's `mintId`); refused once the store stopped or closed. */ + mintId(): number { + this.assertUsable(); + return this.#ids.mint(); + } + + /** The open host's batch, in which an adapter adopts its commits; `undefined` without a host. */ + get batch(): Batch | undefined { + return this.#attachment?.batch; } /** Every recorded table, by SQL name, as of the last commit. */ @@ -170,24 +217,26 @@ export class ChordStore implements Store { return this.#catalog.get(name); } - /** Attaches the open host's `storage_failed` listener; a store attached to an open host is refused (rule 7). */ - attach(listener: (error: unknown) => void): void { - if (this.#listener !== undefined) { + /** Attaches the open host; a store attached to an open host is refused (rule 7). */ + attach(attachment: Attachment): void { + if (this.#attachment !== undefined) { throw new ChordError({ code: "invalid", reason: "args", message: "store: already attached to an open host" }); } - this.#listener = listener; + this.#attachment = attachment; } /** The host closed: the store may serve another one. */ detach(): void { - this.#listener = undefined; + this.#attachment = undefined; } /** * Commits `writes` as one seq when the commit queue reaches it: `fence.admit()` at the step's entry, the writes in - * rule 9's order (retirements, creations, the rest), the outbox commit, `lastSeq`, `COMMIT`, then `fence.adopt` - * before the next step starts. Resolves with the seq. A backend error or a throwing `adopt` stops the store - * (`storage_failed`); a refused admission rejects `released`. `context` counts the step (no abort once queued). + * their order (retirements, creations, the rest), the outbox commit, `lastSeq` and the allocator's value, `COMMIT`, + * then `fence.adopt` before the next step starts. Resolves with the seq. A refused admission rejects `released`. A + * step the backend rolled back after the step's own code threw (a refusal, D4) rejects with that error unchanged and + * the store goes on; any other failure (`BEGIN`, `COMMIT`, `ROLLBACK`, a throwing `adopt`) is an unknown outcome + * that stops the store (`storage_failed`). `context` counts the step (no abort once queued). */ commit(writes: readonly StagedWrite[], fence: CommitFence, context: Context): Promise { try { @@ -200,18 +249,48 @@ export class ChordStore implements Store { return step; } + /** + * An adapter's commit (§1.3): `commit` admitting every write, its adoption run in the attached host's batch + * (`batch.within`: it joins a running batch or opens one), plainly without a host. Durable's `adopt` publishes inside + * it, so Chord sources over Durable's views are marked in the step. + */ + commitInBatch(writes: readonly StagedWrite[], adopt: (seq: number) => void, context: Context): Promise { + const fence: CommitFence = { + admit: () => true, + adopt: (seq) => { + const batch = this.#attachment?.batch; + if (batch === undefined) adopt(seq); + else batch.within(() => adopt(seq), context); + }, + }; + return this.commit(writes, fence, context); + } + /** * Runs `read` on the backend under the guard (rule 11, Durable's `GuardedStorage`): refused once the store stopped * or closed; a backend error fails the store unless `context` was aborted (then `aborted`); a read settling after - * the store stopped rejects `unavailable/storage_failed`. + * the store stopped rejects `unavailable/storage_failed`. `read` returns data and never throws for a request: an + * adapter checks a request from what it returned, after the guard (slice 6b §1.4). */ - async read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#guarded(() => read(this.#db), context); + } + + /** + * `read` for several queries that must see one state (S4, a document from its record and revisions): one backend + * transaction, guarded as `read`. + */ + readTransaction(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#guarded(() => this.#db.transaction(read), context); + } + + async #guarded(run: () => Promise, context: Context): Promise { this.assertUsable(); - const call = (async () => read(this.#db))(); - this.#underway.add(call); + this.#underway++; let result: T; try { - result = await call; + // A read throwing before its first await is the backend's too: inside the `try`. + result = await run(); } catch (error) { this.assertUsable(); const signal = context.abortSignal; @@ -219,27 +298,31 @@ export class ChordStore implements Store { throw new ChordError({ code: "aborted", message: "store: read aborted" }, { cause: signal.reason }); throw this.#fail(error); } finally { - this.#underway.delete(call); + this.#settle(); } this.assertUsable(); return result; } + /** One read or maintenance write settled: `close` goes on once none is underway. */ + #settle(): void { + if (--this.#underway === 0) this.#drained?.(); + } + /** * Runs `write` on the backend outside the commit queue (the outbox's `trim`), in a backend transaction, guarded as * a read without a context: any error stops the store. */ async maintain(write: (executor: SqliteExecutor) => Promise): Promise { this.assertUsable(); - const call = this.#db.transaction(write); - this.#underway.add(call); + this.#underway++; try { - await call; + await this.#db.transaction(write); } catch (error) { this.assertUsable(); throw this.#fail(error); } finally { - this.#underway.delete(call); + this.#settle(); } } @@ -252,7 +335,11 @@ export class ChordStore implements Store { this.#state = { kind: "closed" }; for (const end of [...this.#held]) end(); await this.#tail; - await Promise.allSettled([...this.#underway]); + if (this.#underway > 0) { + await new Promise((resolve) => { + this.#drained = resolve; + }); + } await this.#db.close(); } @@ -273,41 +360,26 @@ export class ChordStore implements Store { const seq = this.#lastSeq + 1; // Past the safe range seqs stop counting exactly (a later one would repeat): the store cannot number further. if (seq > Number.MAX_SAFE_INTEGER) throw this.#fail(new Error("store: no seq past Number.MAX_SAFE_INTEGER")); - const outbox = this.#outbox !== undefined; - let step: Step; + let thrown: unknown = NONE; + let done: { readonly step: Step; readonly nextId: number }; try { - step = await this.#db.transaction(async (executor) => { - const step = new Step(executor, seq, this.#nextDocumentId, this.#catalog); - const recorded: StoreWrite[] | undefined = outbox ? [] : undefined; - for (const order of ORDER) { - for (const write of writes) { - if (write.order !== order) continue; - const done = await write.apply(step); - recorded?.push(done); - } + done = await this.#db.transaction(async (executor) => { + try { + return await this.#write(executor, seq, writes); + } catch (error) { + thrown = error; + throw error; } - await writeCatalog(step); - if (recorded !== undefined) { - await step.run( - `INSERT INTO ${TABLES.outbox} (seq, format, writes) VALUES (?, ?, ?)`, - seq, - OUTBOX_FORMAT, - JSON.stringify(recorded), - ); - } - await step.run( - `UPDATE ${TABLES.meta} SET last_seq = ?, next_document_id = ? WHERE singleton = 1`, - seq, - step.nextDocumentId, - ); - return step; }); } catch (error) { + // The facade rolled back and rejects with the step's own error: a refusal, the store goes on (D4). + if (error === thrown) throw error; throw this.#fail(error); } // `COMMIT` returned: the commit is durable, and memory follows. + const { step, nextId } = done; this.#lastSeq = seq; - this.#nextDocumentId = step.nextDocumentId; + this.#committedNextId = nextId; for (const [name, record] of step.tables) this.#catalog.set(name, record); const metrics = context.telemetry.metrics; metrics?.counter("chord.store.commits").add(1); @@ -320,6 +392,112 @@ export class ChordStore implements Store { return seq; } + /** + * The step's writes inside its backend transaction: each staged write in its phase, the catalog, the outbox commit, + * the metadata. The allocator's value is taken once, after the writes applied (mints meanwhile only raise it). + */ + async #write( + executor: SqliteExecutor, + seq: number, + writes: readonly StagedWrite[], + ): Promise<{ readonly step: Step; readonly nextId: number }> { + const step = new Step(executor, seq, this.#ids, this.#catalog); + const recorded: StoreWrite[] | undefined = this.#outbox !== undefined ? [] : undefined; + for (const order of ORDER) { + for (const write of writes) { + if (write.order !== order) continue; + const done = await write.apply(step); + recorded?.push(...done); + } + } + await writeCatalog(step); + const nextId = this.#ids.value; + if (recorded !== undefined) { + await step.run( + `INSERT INTO ${TABLES.outbox} (seq, format, next_id, writes) VALUES (?, ?, ?, ?)`, + seq, + OUTBOX_FORMAT, + nextId, + JSON.stringify(recorded), + ); + } + await step.run(`UPDATE ${TABLES.meta} SET last_seq = ?, next_id = ? WHERE singleton = 1`, seq, nextId); + return { step, nextId }; + } + + /** + * Batch apply's turn (slice 6b §1.6, S5): refused (`invalid/args`) while an owner holds a claim (its memory would + * go stale) and once the store diverged; otherwise `apply` runs in the commit queue, in one backend transaction, and + * returns the position its last commit leaves (`lastSeq`, the allocator's value) and the tables it changed, which + * memory takes after `COMMIT`. `apply`'s own error rolls back and rejects as a refusal (a `Diverged` one marks the + * store diverged); any other failure stops the store. + */ + replay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + try { + this.#assertReplayable(); + } catch (error) { + return Promise.reject(error); + } + const turn = this.#tail.then(() => this.#replay(apply, isDivergence)); + this.#tail = turn.then(NOTHING, NOTHING); + return turn; + } + + /** At its call and again at its turn: a claim or a divergence may come between the two. */ + #assertReplayable(): void { + this.assertUsable(); + if (this.#diverged) throw invalid("args", "store: diverged; a batch can no longer apply"); + if (this.claims.held) throw invalid("args", "store: an owner holds claims; a batch would stale its memory"); + } + + async #replay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + this.#assertReplayable(); + // While it applies, no owner may claim (and read) what it is changing. + const unlock = this.claims.lock(); + try { + await this.#applyReplay(apply, isDivergence); + } finally { + unlock(); + } + } + + async #applyReplay( + apply: (executor: SqliteExecutor, catalog: ReadonlyMap) => Promise, + isDivergence: (error: unknown) => boolean, + ): Promise { + let thrown: unknown = NONE; + let replayed: Replayed; + try { + replayed = await this.#db.transaction(async (executor) => { + try { + return await apply(executor, this.#catalog); + } catch (error) { + thrown = error; + throw error; + } + }); + } catch (error) { + if (error !== thrown) throw this.#fail(error); + if (isDivergence(error)) this.#diverged = true; + throw error; + } + this.#lastSeq = replayed.lastSeq; + this.#committedNextId = replayed.nextId; + this.#ids.raise(replayed.nextId - 1); + for (const [name, record] of replayed.tables) this.#catalog.set(name, record); + } + + /** Whether the store has an outbox, which a replayed batch writes too. */ + get recordsOutbox(): boolean { + return this.#outbox !== undefined; + } + /** Throws `unavailable/storage_failed` once the store stopped, `released` once it closed. */ assertUsable(): void { const state = this.#state; @@ -332,7 +510,7 @@ export class ChordStore implements Store { #fail(error: unknown): ChordError { if (this.#state.kind === "open") { this.#state = { kind: "failed", error }; - this.#listener?.(error); + this.#attachment?.failed(error); } return storageFailed(error); } diff --git a/packages/chord/src/next/store/table-writes.ts b/packages/chord/src/next/store/table-writes.ts index 101c7f8b3..b517b166b 100644 --- a/packages/chord/src/next/store/table-writes.ts +++ b/packages/chord/src/next/store/table-writes.ts @@ -86,12 +86,12 @@ export type TableDefinitionWrite = Extract applyTable(step, write) }; + return { order: "write", apply: async (step) => [await applyTable(step, write)] }; } /** Stages an `index` write. */ export function stageIndex(write: Extract): StagedWrite { - return { order: "write", apply: (step) => applyIndex(step, write) }; + return { order: "write", apply: async (step) => [await applyIndex(step, write)] }; } /** Stages one edit of the migration of `table` to `version` as its `alter`, `index` or `dropIndex` write. */ @@ -101,11 +101,11 @@ export function stageEdit(table: string, version: number, edit: TableEdit): Stag return stageIndex({ kind: "index", table, name: edit.name, columns: edit.columns }); case "dropIndex": { const write = { kind: "dropIndex", table, name: edit.name } as const; - return { order: "write", apply: (step) => applyDropIndex(step, write) }; + return { order: "write", apply: async (step) => [await applyDropIndex(step, write)] }; } default: { const write = { kind: "alter", table, version, edit } as const; - return { order: "write", apply: (step) => applyAlter(step, write) }; + return { order: "write", apply: async (step) => [await applyAlter(step, write)] }; } } } @@ -117,7 +117,7 @@ export function stagePut(sql: string, table: string, row: EncodedRow, version: n order: "write", apply: async (step) => { await step.run(sql, ...row.params); - return write; + return [write]; }, }; } @@ -167,7 +167,7 @@ export function stageDeleteWhere(statement: Statement, table: string, where: Whe function counted(statement: Statement, record: (count: number) => TableWrite): StagedWrite { return { order: "write", - apply: async (step) => record(await step.runCounted(statement.sql, ...statement.params)), + apply: async (step) => [record(await step.runCounted(statement.sql, ...statement.params))], }; } diff --git a/packages/chord/test/next/bench/jobs.ts b/packages/chord/test/next/bench/jobs.ts new file mode 100644 index 000000000..87cce72d0 --- /dev/null +++ b/packages/chord/test/next/bench/jobs.ts @@ -0,0 +1,164 @@ +// Benchmark of persisted jobs (slice 6b §4, §5): a run's start (its record and state created, resolved after the +// commit) and a state change inside `run` (immediate, and deferred), for a persisted job on a SQLite file against the +// same job in memory. Median and p95 per operation over several bursts. A disk commit is not held to the in-memory +// number: the persisted columns show what a run's records cost on the store. +// Run by hand from packages/chord (not in CI): +// node --conditions=source --experimental-strip-types --expose-gc test/next/bench/jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle, type RunState } from "../../../src/next/jobs/index.ts"; +import { openStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const OPERATIONS = 500; +const RUNS = 7; + +type Progress = { done: number }; + +const InMemoryProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), +}); + +const PersistedProgress = defineState("bench.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 64, +}); + +const InMemoryImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: InMemoryProgress, +}); + +const PersistedImport = defineJob("bench.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: PersistedProgress, + persist: { resume: "rerun" }, +}); + +const Worker = local("bench.worker", { version: 1, params: {}, members: {} }); + +/** Where a run's `run` parks: its state, and the gate that ends it. */ +type Parked = { readonly state: RunState; readonly end: () => void }; + +/** A host serving `Worker` with the job (persisted on a fresh SQLite file, or in memory), its runs parked. */ +async function served(persisted: boolean) { + const directory = await mkdtemp(join(tmpdir(), "chord-bench-jobs-")); + const store = persisted + ? await openStore(sqliteBackend(await openNodeSqliteDatabase(join(directory, "store.sqlite"))), context) + : undefined; + const parked: Parked[] = []; + const waiting: ((parked: Parked) => void)[] = []; + let jobs: JobHandle<{ path: string }, { count: number }, Progress> | undefined; + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Worker, (_, self) => { + jobs = self.job(persisted ? PersistedImport : InMemoryImport, { + run: (_args, ctx) => + new Promise((resolve) => { + const run = { state: ctx.state, end: () => resolve({ count: ctx.state.value.done }) }; + const next = waiting.shift(); + if (next === undefined) parked.push(run); + else next(run); + }), + }); + return {}; + }), + ); + await host.load([{ name: "@bench/jobs", facet }], context); + const reference = await host.get(Worker, {}, context); + return { + jobs: jobs!, + /** The parked `run` of the next run invoked. */ + next: () => new Promise((resolve) => waiting.push(resolve)), + close: async () => { + reference.release(); + await host.close(context); + await store?.close(context); + await rm(directory, { recursive: true, force: true }); + }, + }; +} + +/** Per-operation times (ms) of `RUNS` bursts, after one warm-up burst; `burst` does `OPERATIONS` timed operations. */ +async function bursts( + persisted: boolean, + burst: ( + worker: Awaited>, + time: (operation: () => Promise) => Promise, + ) => Promise, +) { + const times: number[] = []; + for (let run = -1; run < RUNS; run++) { + const worker = await served(persisted); + await burst(worker, async (operation) => { + const before = performance.now(); + await operation(); + if (run >= 0) times.push(performance.now() - before); + }); + await worker.close(); + } + times.sort((a, b) => a - b); + return { median: times[Math.floor(times.length / 2)]!, p95: times[Math.floor(times.length * 0.95)]! }; +} + +/** Starts: each run's start resolved (record and state committed), its `run` parked, all ended after the burst. */ +function starts(persisted: boolean) { + return bursts(persisted, async ({ jobs, next }, time) => { + const parked: Parked[] = []; + for (let index = 0; index < OPERATIONS; index++) { + const invoked = next(); + await time(() => jobs.start({ path: `p${index}` }, context)); + parked.push(await invoked); + } + for (const run of parked) run.end(); + }); +} + +/** State changes: `OPERATIONS` changes of one run's state, immediate or deferred (stored by the outcome). */ +function changes(persisted: boolean, deferred: boolean) { + const options = deferred ? { persistence: { kind: "deferred", withinMs: Infinity } as const } : undefined; + return bursts(persisted, async ({ jobs, next }, time) => { + const invoked = next(); + await jobs.start({ path: "p" }, context); + const { state, end } = await invoked; + for (let index = 0; index < OPERATIONS; index++) { + await time(() => + state.change( + (draft) => { + draft.done = index + 1; + }, + context, + options, + ), + ); + } + end(); + }); +} + +function format({ median, p95 }: { readonly median: number; readonly p95: number }): string { + return `median ${(median * 1000).toFixed(1)} µs, p95 ${(p95 * 1000).toFixed(1)} µs`; +} + +console.log(`start, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await starts(false))}`); +console.log(` persisted: ${format(await starts(true))}`); +console.log(`state change, ${RUNS} bursts of ${OPERATIONS}:`); +console.log(` in memory: ${format(await changes(false, false))}`); +console.log(` persisted, immediate: ${format(await changes(true, false))}`); +console.log(` persisted, deferred: ${format(await changes(true, true))}`); diff --git a/packages/chord/test/next/bench/store.ts b/packages/chord/test/next/bench/store.ts index ce2923de9..e4aa4392c 100644 --- a/packages/chord/test/next/bench/store.ts +++ b/packages/chord/test/next/bench/store.ts @@ -78,7 +78,7 @@ function chordDeltas(outbox: boolean) { const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox, })) as ChordStore; - const address = { namespace: "chord", owner: "session:bench.store", kind: "note", key: "" }; + const address = { namespace: "chord", owner: "bench.store", kind: "note", key: "" }; const base = { version: 1, fingerprint: "0123456789abcdef", value: VALUE }; let id = 0; const fence = { @@ -87,7 +87,7 @@ function chordDeltas(outbox: boolean) { id = ids[0] ?? id; }, }; - await store.commit([createDocument(address, "latest", base, JSON.stringify(VALUE))], fence, context); + await store.commit([createDocument(address, base, JSON.stringify(VALUE))], fence, context); return { commit: (index: number) => { const ops = [["s", ["n"], index + 1]] as const; @@ -149,7 +149,7 @@ function chordPuts(outbox: boolean) { const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox, })) as ChordStore; - const table = tableSqlName("session", "bench.store", Edits.name); + const table = tableSqlName("bench.store", Edits.name); const sql = new TableSql(table, Edits.columns, Edits.rows.physical, Edits.key); const fence = { admit: () => true, adopt: () => {} }; const { version, record: columns, key, fingerprint } = Edits; @@ -329,7 +329,7 @@ console.log(`heap after GC: ${heap() - before} KiB above the start`); async function filledEdits(count: number, body = VALUE.body, definition: typeof Edits = Edits) { const { path, remove } = await file(); const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; - const table = tableSqlName("session", "bench.store", Edits.name); + const table = tableSqlName("bench.store", Edits.name); const fence = { admit: () => true, adopt: () => {} }; await prepareTables([{ definition, table }], store, fence, "bench", context); const handle = new TableHandle(definition, table, store, { diff --git a/packages/chord/test/next/documents/persistence.test.ts b/packages/chord/test/next/documents/persistence.test.ts index 6b97b83a2..13fe83ea8 100644 --- a/packages/chord/test/next/documents/persistence.test.ts +++ b/packages/chord/test/next/documents/persistence.test.ts @@ -27,7 +27,7 @@ let notes: Persistence; beforeEach(async () => { store = await memoryStore({ outbox: true }); - notes = new Persistence(store, "session:app.store", "notes.note"); + notes = new Persistence(store, "app.store", "chord", "notes.note"); }); afterEach(async () => { @@ -47,20 +47,10 @@ async function change( after(draft.state); const prepared = draft.prepare(); const writes: StagedWrite[] = []; - const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes, 0); - let ids: readonly number[] = []; - await store.commit( - writes, - { - admit: () => true, - adopt: (_seq, created) => { - ids = created; - }, - }, - context, - ); + const result = notes.stage(key, definition, stored, { prepared, retires: false }, writes); + await store.commit(writes, { admit: () => true, adopt: () => {} }, context); tracker.adopt(prepared); - return storedAfter(result, ids); + return storedAfter(result); } /** The document at `key` read from the store and decoded by `definition`, `undefined` when absent. */ @@ -259,12 +249,12 @@ describe("staged writes", () => { draft.state.n = n; return draft.prepare(); }; - const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes, 0); - const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes, 1); - const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes, 2); - expect([first, again, gone]).toEqual([ - { kind: "create", version: 1, index: 0 }, - { kind: "create", version: 1, index: 1 }, + const first = notes.stage("b", Note, undefined, { prepared: fresh(2), retires: false }, writes); + const again = notes.stage("a", Note, stored, { prepared: fresh(3), retires: true }, writes); + const gone = notes.stage("c", Note, undefined, { prepared: undefined, retires: true }, writes); + expect([first, again, gone]).toMatchObject([ + { kind: "create", version: 1 }, + { kind: "create", version: 1 }, { kind: "retire" }, ]); let ids: readonly number[] = []; @@ -279,7 +269,7 @@ describe("staged writes", () => { context, ); expect(ids).toEqual([stored!.id + 1, stored!.id + 2]); - expect(storedAfter(again, ids)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); + expect(storedAfter(again)).toEqual({ id: stored!.id + 2, revision: 1, version: 1, deltasSinceBase: 0 }); expect((await outboxKinds()).at(-1)).toEqual(["document.retire", "document.create", "document.create"]); }); @@ -287,6 +277,6 @@ describe("staged writes", () => { // Drafts refuse such values when placed; a prepared change made by hand shows where the text is made. const value = { title: "", n: 1n } as unknown as JsonObject; const prepared = { base: value, value, ops: [], baseRevision: 0, abort: () => {} }; - expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [], 0)).toThrow(TypeError); + expect(() => notes.stage("a", Note, undefined, { prepared, retires: false }, [])).toThrow(TypeError); }); }); diff --git a/packages/chord/test/next/documents/transaction.test.ts b/packages/chord/test/next/documents/transaction.test.ts index ff75ea375..e26052957 100644 --- a/packages/chord/test/next/documents/transaction.test.ts +++ b/packages/chord/test/next/documents/transaction.test.ts @@ -435,7 +435,7 @@ describe("tables in a transaction", () => { order: "write", apply: async () => { await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }, ], @@ -533,6 +533,22 @@ describe("persisted documents in a transaction", () => { await store.close(context); }); + it("a keyed retirement takes 6a's statements: the counted DELETE, the revisions, lastSeq (cost guard)", async () => { + const { telemetry, context: counting, counts } = await metered(); + const { self, k, store, host } = await stored(telemetry); + await self.tx(async (tx) => { + (await tx.draft(k, "a")).n = 1; + }, counting); + const before = { ...counts() }; + await self.tx(async (tx) => { + await tx.retire(k, "a"); + }, counting); + // No read of the record first, no second DELETE of its row; plus the outbox commit. + expect(counts()["chord.store.statements"]! - before["chord.store.statements"]!).toBe(4); + await host.close(context); + await store.close(context); + }); + it("assign the created documents' ids by their order in the commit; retire and recreate in one commit", async () => { const { self, k, store, host } = await stored(); await self.tx(async (tx) => { @@ -548,9 +564,9 @@ describe("persisted documents in a transaction", () => { writes.flatMap((write) => (write.kind === "document.create" ? [[write.record.key, write.record.id]] : [])), ); expect(created).toEqual([ - ["b", 1], - ["a", 2], - ["b", 3], + ["b", 2], + ["a", 3], + ["b", 4], ]); expect(commits[1]!.writes.map(({ kind }) => kind)).toEqual(["document.retire", "document.create"]); // What is stored of `b` is the new record's: the next change is a delta on it. @@ -561,7 +577,7 @@ describe("persisted documents in a transaction", () => { }, context, ); - expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 3 }]); + expect((await store.outbox!.read(2, 10, context))[0]!.writes).toMatchObject([{ kind: "document.change", id: 4 }]); await host.close(context); await store.close(context); }); @@ -644,7 +660,7 @@ describe("persisted documents in a transaction", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); @@ -700,7 +716,7 @@ describe("persisted documents in a transaction", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); diff --git a/packages/chord/test/next/examples/19-jobs.ts b/packages/chord/test/next/examples/19-jobs.ts index f52ee317a..06964dc63 100644 --- a/packages/chord/test/next/examples/19-jobs.ts +++ b/packages/chord/test/next/examples/19-jobs.ts @@ -140,7 +140,7 @@ const facet = defineFacet

((host) => { runShell: async ({ command, lines, requestId }, ctx) => (await shell.start({ command, lines }, ctx, { key: requestId })).id, abortShell: async ({ id }) => { - shell.abort(id); + void shell.abort(id, context); return null; }, }; diff --git a/packages/chord/test/next/examples/28-persisted-jobs.ts b/packages/chord/test/next/examples/28-persisted-jobs.ts new file mode 100644 index 000000000..4750e3ced --- /dev/null +++ b/packages/chord/test/next/examples/28-persisted-jobs.ts @@ -0,0 +1,125 @@ +// Persisted jobs: `persist: { resume }` keeps each unsettled run's record and state document in the object's store. +// A run survives a crash (counted: five in a row end it `restart_limit`), a clean close (written back `stopped`, resumed +// without counting) and its object's retry; `abort` is stored and joins `run`; a state change may be `deferred`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/28-persisted-jobs.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import { defineJob, type JobHandle } from "../../../src/next/jobs/index.ts"; +import { openStore, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +const context = BACKGROUND_CONTEXT; + +// The run's state: persisted, as a persisted job's must be. +const ImportState = defineState("notes.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 16, +}); + +const ImportJob = defineJob("notes.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + // After a crash or a stop, `run` runs again: it must be idempotent (only work since the last state change repeats). + persist: { resume: "rerun" }, +}); + +const Notes = local("notes.store", { + version: 1, + params: {}, + members: { start: method(str(), str()), abort: method(str(), str()) }, +}); + +/** How the current process's `run` behaves: hang (to be cut off) or finish. */ +let behaviour: "hang" | "finish" = "hang"; +let jobs: JobHandle<{ path: string }, { count: number }, { done: number }> | undefined; + +const facet = defineFacet((host) => { + host.serve(Notes, (_, self) => { + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + console.log(` run ${path}: from done = ${ctx.state.value.done}`); + // Progress, deferred: published at once, stored within 100 ms (or with the next immediate change). + await ctx.state.change( + (draft) => { + draft.done += 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 100 } }, + ); + if (behaviour === "hang") + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + console.log(` finish: ${outcome.kind}`, outcome.kind === "done" ? outcome.result : outcome.error.kind); + }, + }); + jobs = imports; + return { + start: async (path, ctx) => (await imports.start({ path }, ctx)).id, + abort: (id, ctx) => imports.abort(id, ctx), + }; + }); +}); + +/** A process: a store over the file and a host serving `Notes`. */ +async function process(path: string): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ + store, + onError: (error) => console.log(" onError:", error.kind), + closeTimeout: 100, + }); + await host.load([{ name: "@example/notes", facet }], context); + return { store, host }; +} + +const settle = () => new Promise((resolve) => setTimeout(resolve, 150)); +const directory = await mkdtemp(join(tmpdir(), "chord-example-28-")); +const path = join(directory, "session.sqlite"); + +// 1. Start a run, then crash: the store closes under the live host. +console.log("1. start, then crash"); +let { store, host } = await process(path); +const id = await (await host.get(Notes, {}, context)).start("inbox", context); +await settle(); +await store.close(context); + +// 2. The next process counts the crash and reruns it (its state as stored: the deferred change was stored in time). +console.log("2. reopen: the run is counted and rerun"); +({ store, host } = await process(path)); +await settle(); + +// 3. A clean close writes the run back `stopped`: the next process resumes it without counting. +console.log("3. clean close, reopen: resumed, uncounted"); +await host.close(context); +await store.close(context); +behaviour = "finish"; +({ store, host } = await process(path)); +await settle(); +console.log(" unsettled runs now:", jobs?.running.value.length); + +// 4. A stored abort: marked, joined. +console.log("4. abort"); +behaviour = "hang"; +const notes = await host.get(Notes, {}, context); +const second = await notes.start("drafts", context); +await settle(); +console.log(" abort:", await notes.abort(second, context)); +await settle(); +console.log(" once settled:", await notes.abort(second, context), "/ the first run:", await notes.abort(id, context)); + +await host.close(context); +await store.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index 73983ae0a..867502320 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -604,7 +604,7 @@ describe("tables", () => { }, ); await host.get(Store, {}, context); - expect(store.table("client:app.store:edits")).toMatchObject({ + expect(store.table("app.store:edits")).toMatchObject({ version: 1, state: "ready", indexes: { by_at: ["at"] }, @@ -717,7 +717,7 @@ describe("eager migrations at activation (slice 6a §9)", () => { afterEach(() => { pumping = false; }); - const TABLE = "client:app.store:edits"; + const TABLE = "app.store:edits"; const OPEN = { admit: () => true, adopt: () => {} }; const EditsV1 = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); diff --git a/packages/chord/test/next/host/follow.test.ts b/packages/chord/test/next/host/follow.test.ts new file mode 100644 index 000000000..de225e098 --- /dev/null +++ b/packages/chord/test/next/host/follow.test.ts @@ -0,0 +1,345 @@ +// Tests of `self.follow` (slice 6b §3, D9, R4): a source over another runtime's synchronous observer, attached during +// activation, its calls marked in the running batch or grouped into one batch outside it, its ends. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Op } from "../../../src/delta/index.ts"; +import { int, local, obj } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { FollowSource } from "../../../src/next/host/follow.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { FollowAttach, FollowObserver } from "../../../src/next/host/index.ts"; +import { Batch } from "../../../src/next/state/batch.ts"; +import type { Watchable, WatchEnd } from "../../../src/watch.ts"; +import { changesOf, delivered, expectCollected, instancesOf, loaded, valuesOf } from "../support.ts"; + +const followed = instancesOf(FollowSource, "attach"); + +type Count = { readonly n: number }; + +/** A runtime with synchronous observers, as Durable's views have them. */ +class Runtime { + value: Count = { n: 0 }; + readonly observers = new Set>(); + attachments = 0; + detached = 0; + /** Resolves the next attachment when set (an attachment that takes time). */ + gate: Promise | undefined; + absent = false; + + readonly attach: FollowAttach = async (observer) => { + this.observers.add(observer); + this.attachments++; + await this.gate; + if (this.absent) return undefined; + return { + value: this.value, + detach: () => { + this.observers.delete(observer); + this.detached++; + }, + }; + }; + + /** One change: every observer advanced with its ops. */ + set(n: number, retired = false): void { + const ops: Op[] = [["s", ["n"], n]]; + this.value = { n }; + for (const observer of this.observers) + observer.advance({ kind: "value", value: this.value, ops, retired }, context); + } + + end(end: WatchEnd): void { + for (const observer of [...this.observers]) observer.end(end); + this.observers.clear(); + } +} + +const View = local("app.view", { version: 1, params: {}, members: { a: obj({ n: int() }), sum: int() } }); + +/** Serves `View`, following `a` and `b`, its `sum` a derive over both (its computes counted). */ +async function served(a = new Runtime(), b = new Runtime()) { + let computes = 0; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + const right = self.follow(b.attach); + const sum = self.derive([left, right], ([x, y]) => { + computes++; + return x.n + y.n; + }); + return { a: left, sum }; + }), + ); + const view = await host.get(View, {}, context); + const batch = (host as unknown as ChordHost).batch; + return { host, errors, view, a, b, batch, computes: () => computes }; +} + +describe("calls", () => { + it("inside a running batch: one frame per commit, and a derive over two views computes once with both", async () => { + const { view, a, b, batch, computes } = await served(); + const calls = changesOf(await view.a.watch(context)); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + batch.commit(() => { + a.set(1); + a.set(2); + b.set(10); + }, context); + await delivered(); + expect(calls).toEqual([ + { + kind: "value", + value: { n: 2 }, + ops: [ + ["s", ["n"], 1], + ["s", ["n"], 2], + ], + retired: false, + }, + ]); + expect(valuesOf(sums)).toEqual([12]); + expect(computes() - before).toBe(1); + }); + + it("outside any batch: queued and applied in one batch on the next microtask", async () => { + const { view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1); + b.set(2); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + expect(view.a.value).toEqual({ n: 1 }); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + }); + + it("before the attachment resolved: applied in order on its value", async () => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + const serving = served(a); + await delivered(); + a.set(5); + a.set(6); + gate.resolve(); + const { view } = await serving; + expect(view.a.value).toEqual({ n: 6 }); + }); +}); + +describe("types", () => { + it("infers the value type from the attachment, without a type argument", async () => { + const runtime = new Runtime(); + // A runtime's own observe, as Durable's `observeConversation` is: typed by its value, not by Chord's observer. + const observe = (observer: FollowObserver) => runtime.attach(observer, context); + await loaded((h) => + h.serve(View, (_, self) => { + const a = self.follow((observer) => observe(observer)); + expectTypeOf(a).toEqualTypeOf>(); + return { a, sum: self.derive([a], ([x]) => x.n) }; + }), + ); + }); +}); + +describe("activation", () => { + it("an absent address fails the activation transient; a failed activation detaches what attached", async () => { + const a = new Runtime(); + const b = new Runtime(); + b.absent = true; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => ({ a: self.follow(a.attach), sum: self.derive([self.follow(b.attach)], () => 0) })), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow("followed address is absent"); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await delivered(); + expect([a.attachments, a.detached]).toEqual([a.attachments, a.attachments]); + await host.close(context); + }); +}); + +describe("ends (R4)", () => { + it("failed: the live object fails transient once, reported, and its retry attaches again", async () => { + const { host, errors, view, a, b } = await served(); + a.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + b.end({ kind: "failed", error: new ChordError({ code: "unavailable", message: "session failed" }) }); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + expect((view.state.value as { retryAt?: number }).retryAt).toBeGreaterThan(0); + expect(errors.filter((error) => error.kind === "object_failed")).toHaveLength(1); + expect(view.a.value).toEqual({ n: 0 }); + await delivered(); + view.retry(); + await delivered(); + await delivered(); + expect(view.state.value).toEqual({ kind: "live" }); + expect([a.attachments, a.detached, b.attachments]).toEqual([2, 1, 2]); + a.set(3); + await delivered(); + expect(view.a.value).toEqual({ n: 3 }); + await host.close(context); + }); + + it("ended: final; a queued retirement reaches the watch first, the value stays, a new watch ends at once", async () => { + const { host, errors, view, a } = await served(); + const watch = await view.a.watch(context); + const calls = changesOf(watch); + a.set(4, true); + a.end({ kind: "ended" }); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toMatchObject([{ kind: "value", value: { n: 4 }, retired: true }]); + expect(view.a.value).toEqual({ n: 4 }); + const late = await view.a.watch(context); + expect(late.value).toEqual({ n: 4 }); + expect(await late.closed).toEqual({ kind: "ended" }); + expect(errors).toEqual([]); + expect(view.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("ends during activation and in one batch (review regressions)", () => { + it("an end before the attachment resolved keeps the attachment's value on the source", async () => { + const source = new FollowSource( + async (observer) => { + observer.end({ kind: "ended" }); + return { value: { n: 7 }, detach() {} }; + }, + { label: "early end", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + expect(source.value).toEqual({ n: 7 }); + }); + + it.each([ + ["ended", { kind: "ended" } as const], + ["failed", { kind: "failed", error: new Error("early failure") } as const], + ])("an %s end before the attachment resolved fails the activation transient", async (_, end) => { + const a = new Runtime(); + const gate = Promise.withResolvers(); + a.gate = gate.promise; + // Loading activates the object (local, empty params): its follow attaches meanwhile. + const loading = loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + return { a: left, sum: self.derive([left], ([x]) => x.n) }; + }), + ); + await delivered(); + a.end(end); + gate.resolve(); + const { host, errors } = await loading; + await delivered(); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + expect(a.detached).toBe(a.attachments); + }); + + it("an end while the object activates (after its sources attached) fails the activation transient", async () => { + const a = new Runtime(); + let ended = false; + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + const left = self.follow(a.attach); + // The activation computes the derives after the follows attached: the runtime ends then. + const sum = self.derive([left], ([x]) => { + if (!ended) { + ended = true; + a.end({ kind: "ended" }); + } + return x.n; + }); + return { a: left, sum }; + }), + ); + await expect(host.get(View, {}, context, { readyTimeout: 50 })).rejects.toThrow(); + expect(ended).toBe(true); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + failure: { kind: "transient" }, + }); + await host.close(context); + }); + + it("a retirement and an end inside one batch: the watch gets the retirement, then ends", async () => { + const batch = new Batch(() => {}, undefined); + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 7 }, detach() {} }; + }, + { label: "retirement", context, ended() {} }, + batch, + ); + await source.attach(context); + const watch = await source.watch(context); + const calls = changesOf(watch); + batch.commit(() => { + observer!.advance({ kind: "value", value: { n: 7 }, ops: [], retired: true }, context); + observer!.end({ kind: "ended" }); + }, context); + expect(await watch.closed).toEqual({ kind: "ended" }); + expect(calls).toEqual([{ kind: "value", value: { n: 7 }, ops: [], retired: true }]); + }); + + it("a call and an end outside a batch with nothing watching: a later watch opens at the value and ends", async () => { + let observer: FollowObserver | undefined; + const source = new FollowSource( + async (attached) => { + observer = attached; + return { value: { n: 0 }, detach() {} }; + }, + { label: "unwatched", context, ended() {} }, + new Batch(() => {}, undefined), + ); + await source.attach(context); + observer!.advance({ kind: "value", value: { n: 1 }, ops: [["s", ["n"], 1]], retired: true }, context); + observer!.end({ kind: "ended" }); + await delivered(); + const watch = await source.watch(context); + expect(watch.value).toEqual({ n: 1 }); + expect(await watch.closed).toEqual({ kind: "ended" }); + source.detach(); + }); + + it("an end outside a batch after calls of two views: the views of one commit are delivered together", async () => { + const { host, view, a, b, computes } = await served(); + const sums = changesOf(await view.sum.watch(context)); + const before = computes(); + a.set(1, true); + b.set(2); + a.end({ kind: "ended" }); + await delivered(); + expect(valuesOf(sums)).toEqual([3]); + expect(computes() - before).toBe(1); + await host.close(context); + }); +}); + +describe("GC", () => { + it("a source its runtime ended is still detached at the object's end, then collected", async () => { + const a = new Runtime(); + const { host } = await served(a); + a.end({ kind: "ended" }); + await delivered(); + expect(a.detached).toBe(0); + await host.close(context); + expect(a.detached).toBe(1); + await expectCollected(...followed.splice(0)); + }); + + it("an ended object's followed sources are collected, and their observers detached", async () => { + const a = new Runtime(); + const { host } = await served(a); + await host.close(context); + expect(a.observers.size).toBe(0); + await expectCollected(...followed.splice(0)); + }); +}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 96d2482c4..07c4eb751 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -21,6 +21,7 @@ import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { failures, fakeTimers, + faultyStore, flushed, forEachCodec, linked, @@ -921,20 +922,16 @@ describe("the host's store", () => { }); it("reports storage_failed once and closes the host; the app still closes the store", async () => { - const store = await memoryStore(); + const { store, fail } = await faultyStore(); const errors: HostError[] = []; const host = createHost({ store, onError: (error) => errors.push(error) }); await host.load([], context); - const broken: StagedWrite = { - order: "write", - apply: async (step) => { - await step.run("INSERT INTO no_such_table VALUES (1)"); - return { kind: "document.retire", id: 0, revision: 0 }; - }, - }; + const write: StagedWrite = { order: "write", apply: async () => [] }; const fence = { admit: () => true, adopt: () => {} }; - await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); - await expect(store.commit([broken], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + // A failed `COMMIT`: an unknown outcome (D4). + fail("COMMIT"); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await expect(store.commit([write], fence, context)).rejects.toMatchObject({ reason: "storage_failed" }); // The host closed itself; the app's own close returns that closing. expect((host as ChordHost).closing).toBe(true); await host.close(context); diff --git a/packages/chord/test/next/host/member-source.test.ts b/packages/chord/test/next/host/member-source.test.ts index 77939396b..4a804cdb7 100644 --- a/packages/chord/test/next/host/member-source.test.ts +++ b/packages/chord/test/next/host/member-source.test.ts @@ -386,6 +386,7 @@ describe("subscribers, directly", () => { advance: (call) => void events.push(call.kind === "value" ? [call.value, call.ops] : (call.error as Error).message), end: (end) => void events.push(end.kind), + endAfterDelivery: () => void events.push("ended"), }; expect(source.value).toBe(0); source.subscribe(subscriber); diff --git a/packages/chord/test/next/jobs/definition.test.ts b/packages/chord/test/next/jobs/definition.test.ts index 4f16502ac..aea86122d 100644 --- a/packages/chord/test/next/jobs/definition.test.ts +++ b/packages/chord/test/next/jobs/definition.test.ts @@ -96,11 +96,34 @@ describe("defineJob", () => { ); }); - it("refuses persist before slice 6, in its final shape", () => { - expect(() => defineJob("j", { version: 1, args, result: int(), persist: { resume: "rerun" } })).toThrow( - "j.persist: persisted jobs are not supported before slice 6", + it("takes persist with a resume rule, a persisted state and a JSON-only result (slice 6b §4)", () => { + const Persisted = defineState("p.state", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const InMemory = defineState("m.state", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const job = defineJob("j", { version: 1, args, result: int(), state: Persisted, persist: { resume: "rerun" } }); + expect(job.persist).toEqual({ resume: "rerun" }); + expect(() => + defineJob("j", { version: 1, args, result: int(), state: InMemory, persist: { resume: "abandon" } }), + ).toThrow("a persisted job's state is a persisted document"); + expect(() => defineJob("j", { version: 1, args, result: bytes(), persist: { resume: "rerun" } })).toThrow( + "a persisted job's result is JSON", ); + // An in-memory job may return bytes. + expect(() => defineJob("j", { version: 1, args, result: bytes() })).not.toThrow(); // @ts-expect-error: resume is required - expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow(); + expect(() => defineJob("j", { version: 1, args, result: int(), persist: {} })).toThrow( + 'expected "rerun" or "abandon"', + ); + }); + + it("fingerprints args, result and state: a schema change without a version bump changes it", () => { + const one = defineJob("j", { version: 1, args, result: int() }); + expect(defineJob("j", { version: 1, args, result: int() }).fingerprint).toBe(one.fingerprint); + expect(defineJob("j", { version: 1, args, result: str() }).fingerprint).not.toBe(one.fingerprint); }); }); diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts index a868d0f04..4fd9aadcb 100644 --- a/packages/chord/test/next/jobs/job.test.ts +++ b/packages/chord/test/next/jobs/job.test.ts @@ -325,23 +325,27 @@ describe("start", () => { expect(log.value.list).toEqual(["written with the start", "from run a"]); }); - it("starts run with its signal aborted after an abort between the commit and its start", async () => { + it("an abort between the commit and the start is marked, then the signal fires; the abort joins run (D7)", async () => { let signalled: boolean | undefined; const { job } = await declared(Echo, () => ({ run: async ({ text }, ctx) => { + await new Promise((resolve) => setTimeout(resolve, 20)); signalled = ctx.abortSignal?.aborted; return text; }, })); const watch = await job.running.watch(context); + const aborts: Promise[] = []; // The watch's frame of the start's commit arrives before `run` starts. watch.start(async (call) => { - if (call.kind !== "value") return; - for (const entry of call.value) job.abort(entry.id); + if (call.kind !== "value" || aborts.length > 0) return; + for (const entry of call.value) aborts.push(job.abort(entry.id, context)); }); const run = await job.start({ text: "a" }, context); expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "a" }); expect(signalled).toBe(true); + expect(await Promise.all(aborts)).toEqual(["marked"]); + expect(await job.abort(run.id, context)).toBe("terminal"); await watch.stop(); }); @@ -578,7 +582,7 @@ describe("run", () => { })); const run = await job.start({ command: "a" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); expect(await run.outcome(context)).toEqual({ id: run.id, args: { command: "a" }, @@ -598,7 +602,7 @@ describe("run", () => { })); const run = await job.start({ text: "kept" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); expect(await run.outcome(context)).toMatchObject({ kind: "done", result: "kept" }); }); @@ -779,7 +783,7 @@ describe("finish", () => { const other = await job.start({ command: "c" }, context, { key: "other" }); await vi.advanceTimersByTimeAsync(0); expect(job.running.value.map((entry) => entry.phase)).toEqual(["blocked", "finishing"]); - job.abort(run.id); + void job.abort(run.id, context); await vi.advanceTimersByTimeAsync(0); expect(await dropped).toMatchObject({ code: "aborted" }); expect(job.running.value.map((entry) => entry.id)).toEqual([other.id]); @@ -819,7 +823,7 @@ describe("finish", () => { })); const run = await job.start({ text: "a" }, context); await flushed(); - job.abort(run.id); + void job.abort(run.id, context); gate.resolve(); expect((await run.outcome(context)).kind).toBe("done"); }); @@ -1299,7 +1303,7 @@ describe("GC", () => { () => new Error("expected a rejection"), (error: Error) => error, ); - job.abort(run.id); + void job.abort(run.id, context); await vi.advanceTimersByTimeAsync(0); expect(await dropped).toMatchObject({ code: "aborted" }); expect(job.running.value).toEqual([]); diff --git a/packages/chord/test/next/jobs/persisted.test.ts b/packages/chord/test/next/jobs/persisted.test.ts new file mode 100644 index 000000000..80751d207 --- /dev/null +++ b/packages/chord/test/next/jobs/persisted.test.ts @@ -0,0 +1,1066 @@ +// Tests of persisted jobs (slice 6b §4, D5-D8, R3): run records and state documents in the object's store, every +// transition of §4.2 across a crash (the store closed under a live host, reopened by a new one), the clean stop and +// the close writer, a seal and its retry, the restart limit and its reset, the finish failure count, a stored abort, +// deferred state. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { MAX_PENDING_OPS } from "../../../src/delta/pending.ts"; +import { arr, int, local, num, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { RETENTION_MS, Residency } from "../../../src/next/documents/residency.ts"; +import { createHost, defineFacet, type HostError, type StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineJob, + type JobHandle, + type JobImplementation, + type Outcome, + type RunState, +} from "../../../src/next/jobs/index.ts"; +import { JobRun } from "../../../src/next/jobs/run.ts"; +import { sqliteBackend } from "../../../src/next/store/backend.ts"; +import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { delivered, expectCollected, failLive, instancesOf } from "../support.ts"; + +const attempted = instancesOf(ChordObject, "attempt"); + +const Progress = defineState("import.progress", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 8, +}); + +const Import = defineJob("notes.import", importOptions()); + +/** `Import`'s declaration, which the other versions below change. */ +function importOptions() { + return { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: Progress, + persist: { resume: "rerun" }, + } as const; +} + +/** `Import` at the next version, as a deploy changes it. */ +const ImportV2 = defineJob("notes.import", { ...importOptions(), version: 2 }); + +/** `Import` at its version with another result schema (`num` for `int`): another fingerprint. */ +const ImportReshaped = defineJob("notes.import", { ...importOptions(), result: obj({ count: num() }) }); + +const Abandoned = defineJob("notes.abandoned", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + persist: { resume: "abandon" }, +}); + +const Store = local("notes.store", { version: 1, params: {}, members: {} }); + +type ImportArgs = { readonly path: string }; +type ImportResult = { readonly count: number }; +type ImportOutcome = Outcome; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A store file opened by `open()` again after each crash or close. */ +async function storeFile() { + directory = await mkdtemp(join(tmpdir(), "chord-jobs-")); + const path = join(directory, "store.sqlite"); + return async () => (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; +} + +/** What one host over the store sees of the store object's jobs. */ +type Opened = { + readonly store: ChordStore; + readonly errors: HostError[]; + readonly imports: JobHandle; + readonly abandoned: JobHandle; + readonly finished: ImportOutcome[]; + readonly self: StoreSelf; + /** `ref.retry()` of the store object. */ + retry(): Promise; + close(): Promise; + /** A crash: the store's backend closes under the live host, which is left behind. */ + crash(): Promise; +}; + +/** A host over `store` serving `Store` with both jobs (`Import` or another declaration of it), `run` and `finish` given. */ +async function opened( + store: ChordStore, + run: JobImplementation["run"], + finish?: (outcome: ImportOutcome) => void, + declared: typeof Import = Import, +): Promise { + const errors: HostError[] = []; + const finished: ImportOutcome[] = []; + const host = createHost({ store, onError: (error) => errors.push(error), closeTimeout: 200 }); + let found: Pick | undefined; + await host.load( + [ + { + name: "notes", + facet: defineFacet((h) => + h.serve(Store, (_, self) => { + const imports = self.job(declared, { + run, + finish: async (outcome) => { + finish?.(outcome); + finished.push(outcome); + }, + }); + const abandoned = self.job(Abandoned, { + run: async () => ({ count: 0 }), + finish: async (outcome) => void finished.push(outcome as unknown as ImportOutcome), + }); + found = { imports, abandoned, self }; + return {}; + }), + ), + }, + ], + context, + ); + return { + store, + errors, + finished, + ...found!, + retry: async () => (await host.hold(Store, {}, context)).retry(), + close: async () => { + await host.close(context); + await store.close(context); + }, + crash: () => store.close(context), + }; +} + +/** The stored revisions of the run state documents, in seq order: their kinds and contents. */ +async function stateRevisions(store: ChordStore): Promise<[string, unknown][]> { + const rows = await store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + `SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.job-states' ORDER BY r.seq`, + ), + context, + ); + return rows.map(({ kind, content }) => [kind, JSON.parse(content)]); +} + +/** Waits `ms` of real time (a drain's bound passing, a wait that must not settle). */ +function sleepMs(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** The records the store keeps of `namespace`'s documents: their kinds and keys. */ +async function stored(store: ChordStore, namespace: string): Promise { + return store.read( + (executor) => executor.all("SELECT kind, key FROM chord_documents WHERE namespace = ? ORDER BY id", namespace), + context, + ); +} + +/** The current record of run `id`. */ +async function record(store: ChordStore, id: string): Promise | undefined> { + const row = await store.read( + (executor) => + executor.get<{ content: string }>( + `SELECT r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id + WHERE d.namespace = 'chord.jobs' AND d.key = ? ORDER BY r.seq DESC LIMIT 1`, + id, + ), + context, + ); + return row === undefined ? undefined : (JSON.parse(row.content) as Record); +} + +describe("records", () => { + it("a run's record and state live in the store while it is unsettled; finish deletes the record, retires the state", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context, { key: "k" }); + await delivered(); + expect(await stored(jobs.store, "chord.jobs")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([{ kind: "notes.import", key: run.id }]); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "running", + restarts: 0, + key: "k", + args: { path: "a" }, + }); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 }, state: { done: 3 } }); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("crashes and restarts (D6)", () => { + it("a run cut off while running is counted and rerun; a state change resets the count; five in a row end it", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + for (let restart = 1; restart < 5; restart++) { + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: restart }); + } + await jobs.crash(); + jobs = await opened(await open(), hang); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "restart_limit", restarts: 5 } }]); + expect(jobs.errors).toContainEqual( + expect.objectContaining({ kind: "job_interrupted", job: "notes.import", id: run.id, restarts: 5 }), + ); + await jobs.close(); + }); + + it("a state change resets the restart count, stored at once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred" } }, + ); + return new Promise(() => {}); + }); + await delivered(); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 0 }); + await jobs.crash(); + }); + + it("abandon: a run cut off by a crash finishes failed { aborted, restart } without run", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async () => ({ count: 0 })); + const gate = Promise.withResolvers(); + // Its run never returns before the crash. + void gate; + const run = await jobs.abandoned.start({ path: "a" }, context); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 0 })); + await delivered(); + await delivered(); + expect(jobs.finished.filter((outcome) => outcome.id === run.id)).toMatchObject([ + { kind: "failed", error: { kind: "aborted", reason: "restart" } }, + ]); + await jobs.close(); + }); +}); + +describe("versions", () => { + it.each([ + ["another version", ImportV2], + ["the same version with another fingerprint", ImportReshaped], + ])("a run stored by %s is blocked (version): kept as stored, never run, listed blocked", async (_, declared) => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + let runs = 0; + jobs = await opened( + await open(), + async () => { + runs++; + return { count: 1 }; + }, + undefined, + declared, + ); + await delivered(); + expect(runs).toBe(0); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // Kept as stored: the crash is not counted, slice 7 maps or resolves it. + expect(await record(jobs.store, run.id)).toMatchObject({ version: 1, phase: "running", restarts: 0 }); + await jobs.close(); + }); +}); + +describe("versions with incompatible state (review regression)", () => { + it("a run of another version is blocked before its state is decoded: no migration, no activation failure", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const changed = defineState("import.progress", { + schema: obj({ done: str() }), + version: 2, + initial: () => ({ done: "new" }), + persist: true, + checkpointWhen: () => false, + migrate: () => { + throw new Error("must not migrate an unsupported run"); + }, + }); + const declared = defineJob("notes.import", { ...importOptions(), version: 2, state: changed }); + // The declaration's state type differs from `Import`'s: this host only lists the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, declared as unknown as typeof Import); + expect(jobs.errors).toEqual([]); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + // An abort drops it: its record and its undecoded state document are retired. + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("versions without a state (review regression)", () => { + it("an abort of a run blocked under a version without a state retires the old state document too", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const stateless = defineJob("notes.import", { ...importOptions(), version: 2, state: undefined }); + // The declaration's state type differs from `Import`'s: this host only lists and drops the blocked run. + jobs = await opened(await open(), async () => ({ count: 0 }), undefined, stateless as unknown as typeof Import); + expect(jobs.imports.running.value).toEqual([{ id: run.id, version: 1, phase: "blocked", reason: "version" }]); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await stored(jobs.store, "chord.jobs")).toEqual([]); + expect(await stored(jobs.store, "chord.job-states")).toEqual([]); + await jobs.close(); + }); +}); + +describe("a seal during the reservation (review regressions)", () => { + it("a reservation refused at admission after the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The second commit is the reservation: it reaches the store only after the seal. + jobs.store.commit = (writes, fence, ctx) => { + if (++commits !== 2) return commit(writes, fence, ctx); + entered.resolve(); + return gate.promise.then(() => commit(writes, fence, ctx)); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal during reservation" })); + await jobs.retry(); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("a reservation queued behind a held commit queue past the seal: the next live attempt invokes the run", async () => { + const open = await storeFile(); + let invoked = 0; + const jobs = await opened(await open(), async () => ({ count: ++invoked })); + const held = Promise.withResolvers<() => void>(); + const commit = jobs.store.commit.bind(jobs.store); + let first = true; + // The start's commit takes the queue as it adopts (an export would): the reservation waits behind it. + jobs.store.commit = (writes, fence, ctx) => { + if (!first) return commit(writes, fence, ctx); + first = false; + const holding = { + admit: () => fence.admit(), + adopt: (seq: number, ids: readonly number[]) => { + fence.adopt(seq, ids); + void jobs.store.take(() => {}).then(held.resolve); + }, + }; + return commit(writes, holding, ctx); + }; + const run = await jobs.imports.start({ path: "p" }, context); + const release = await held.promise; + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal queued reservation" })); + await jobs.retry(); + release(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 1 } }); + expect(invoked).toBe(1); + await jobs.close(); + }); + + it("an abort joined to a reservation that commits after the seal returns marked", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 1 })); + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = jobs.store.commit.bind(jobs.store); + let commits = 0; + // The reservation's commit is admitted, then held past the seal (its continuation sees the attempt sealed). + jobs.store.commit = (writes, fence, ctx) => { + const committed = commit(writes, fence, ctx); + if (++commits !== 2) return committed; + entered.resolve(); + return committed.then(async (seq) => { + await gate.promise; + return seq; + }); + }; + const run = await jobs.imports.start({ path: "p" }, context); + await entered.promise; + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "seal after reservation" })); + gate.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); +}); + +describe("the clean stop and the close writer", () => { + it("host close writes the stopped runs back: the next start resumes them without counting", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + (_args, ctx) => + new Promise((_, reject) => ctx.abortSignal?.addEventListener("abort", () => reject(new Error("stopped")))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.close(); + const reopened = await open(); + expect(await record(reopened, run.id)).toMatchObject({ phase: "stopped", restarts: 0 }); + jobs = await opened(reopened, async () => ({ count: 7 })); + await delivered(); + await delivered(); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 7 } }]); + await jobs.close(); + }); + + it("a seal (a live failure) stops the run; the retry resumes it once its old invocation settled, uncounted (R3)", async () => { + const open = await storeFile(); + let invocations = 0; + const old = Promise.withResolvers(); + const jobs = await opened(await open(), async () => { + invocations++; + if (invocations === 1) { + // Ignores its signal until released. + await old.promise; + return { count: -1 }; + } + return { count: invocations }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + await delivered(); + // The retry waits for the old invocation (the drain, bounded by closeTimeout): no second run beside it. + expect(invocations).toBe(1); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + expect(jobs.finished.map(({ kind }) => kind)).toEqual(["done"]); + await jobs.close(); + }); + + it("a close writer that cannot commit leaves the records running: the next start counts a restart", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + await jobs.close(); + jobs = await opened(await open(), () => new Promise(() => {})); + await delivered(); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.crash(); + }); +}); + +describe("invocations after a seal (R3, review regressions)", () => { + /** A store object whose first invocation ignores its signal until `old` resolves, the next ones run `next`. */ + async function sealed(next: (ctx: { readonly abortSignal?: AbortSignal }) => Promise) { + const open = await storeFile(); + const old = Promise.withResolvers(); + const signals: boolean[] = []; + let invocations = 0; + const jobs = await opened(await open(), async (_args, ctx) => { + signals.push(ctx.abortSignal?.aborted ?? false); + if (++invocations > 1) return next(ctx); + await old.promise; + const wrote = await ctx.state + .change((draft) => { + draft.done = 99; + }, ctx) + .then( + () => "accepted", + (error: unknown) => error, + ); + return { count: wrote === "accepted" ? -2 : -1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + return { jobs, run, old, signals, invocations: () => invocations }; + } + + it("an invocation replaced after the bounded drain writes nothing and records nothing", async () => { + const current = Promise.withResolvers(); + const { jobs, run, old, invocations } = await sealed(async () => { + await current.promise; + return { count: 2 }; + }); + await jobs.retry(); + // The drain (closeTimeout 200 ms) ends without the old invocation: the retry invokes the run again. + await sleepMs(300); + expect(invocations()).toBe(2); + old.resolve(); + await delivered(); + await delivered(); + expect(jobs.finished).toEqual([]); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + current.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 2 } }); + await jobs.close(); + }); + + it("a resumed invocation gets a fresh abort signal", async () => { + const { jobs, run, old, signals } = await sealed(async () => ({ count: 1 })); + await jobs.retry(); + old.resolve(); + await run.outcome(context); + expect(signals).toEqual([false, false]); + await jobs.close(); + }); + + it("abort joins the resumed invocation, not the first", async () => { + const second = Promise.withResolvers(); + const { jobs, run, old } = await sealed(async () => { + await second.promise; + return { count: 2 }; + }); + old.resolve(); + await jobs.retry(); + await sleepMs(20); + let settled = false; + const aborting = jobs.imports.abort(run.id, context).then((answer) => { + settled = true; + return answer; + }); + await sleepMs(50); + expect(settled).toBe(false); + second.resolve(); + expect(await aborting).toBe("marked"); + await jobs.close(); + }); + + it("resume abandon: a run the seal stopped finishes failed { aborted, restart } without a second run", async () => { + const open = await storeFile(); + const old = Promise.withResolvers(); + let calls = 0; + const declared = defineJob("notes.import", { ...importOptions(), persist: { resume: "abandon" } }); + const jobs = await opened( + await open(), + async () => { + calls++; + if (calls === 1) await old.promise; + return { count: calls }; + }, + undefined, + declared, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + failLive(attempted, jobs.self, new ChordError({ code: "unavailable", message: "boom" })); + await jobs.retry(); + old.resolve(); + expect(await run.outcome(context)).toMatchObject({ + kind: "failed", + error: { kind: "aborted", reason: "restart" }, + }); + expect(calls).toBe(1); + await jobs.close(); + }); +}); + +describe("loading (review regressions)", () => { + it("a refused recovery commit loses nothing: the retry loads the runs again and counts the crash once", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const store = await open(); + const refused = vi.spyOn(store, "commit").mockRejectedValueOnce(new Error("refused recovery")); + let invocations = 0; + jobs = await opened(store, () => { + invocations++; + return new Promise(() => {}); + }); + expect(jobs.errors).toContainEqual(expect.objectContaining({ kind: "activation_failed" })); + refused.mockRestore(); + await jobs.retry(); + await delivered(); + await delivered(); + expect(invocations).toBe(1); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id }]); + expect(await record(store, run.id)).toMatchObject({ phase: "running", restarts: 1 }); + await jobs.close(); + }); + + it("a state document that fails to load fails the activation; the retry with fixed code loads the run", async () => { + const open = await storeFile(); + const hang = (): Promise => new Promise(() => {}); + const failing = { fail: true }; + const ProgressV2 = defineState("import.progress", { + schema: obj({ done: int() }), + version: 2, + migrate: (old) => { + if (failing.fail) throw new Error("migrate failed"); + return old as { done: number }; + }, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, + }); + let jobs = await opened(await open(), hang); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + jobs = await opened( + await open(), + hang, + undefined, + defineJob("notes.import", { ...importOptions(), state: ProgressV2 }), + ); + expect(jobs.errors.map(({ kind }) => kind)).toContain("activation_failed"); + failing.fail = false; + await jobs.retry(); + await delivered(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 0 } }]); + await jobs.close(); + }); +}); + +describe("finish failures", () => { + it("are counted on the record, across a crash", async () => { + const open = await storeFile(); + let jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("finish failed"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 1 }); + await jobs.crash(); + const failing = () => { + throw new Error("finish failed"); + }; + jobs = await opened(await open(), async () => ({ count: 1 }), failing); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing", finishFailures: 2 }); + await jobs.crash(); + jobs = await opened(await open(), async () => ({ count: 1 })); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", result: { count: 1 } }]); + await jobs.close(); + }); +}); + +describe("abort (D7)", () => { + it("is stored and joined: marked once run settled, terminal for an unknown id", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + (_args, ctx) => + new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", () => resolve({ count: -1 }))), + ); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: -1 } }); + expect(await jobs.imports.abort(run.id, context)).toBe("terminal"); + await jobs.close(); + }); + + it("an abort-marked run not invoked yet finishes aborted without run after a crash", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + const aborting = jobs.imports.abort(run.id, context); + await delivered(); + await jobs.crash(); + void aborting.catch(() => {}); + let ran = false; + jobs = await opened(await open(), async () => { + ran = true; + return { count: 1 }; + }); + await delivered(); + await delivered(); + expect(ran).toBe(false); + expect(jobs.finished).toMatchObject([{ kind: "failed", error: { kind: "aborted", reason: "abort" } }]); + await jobs.close(); + }); +}); + +describe("abort of a finishing run (review regression)", () => { + it("stores the mark and keeps the recorded outcome", async () => { + const open = await storeFile(); + const jobs = await opened( + await open(), + async () => ({ count: 1 }), + () => { + throw new Error("retry finish"); + }, + ); + const run = await jobs.imports.start({ path: "a" }, context); + await sleepMs(50); + expect(await record(jobs.store, run.id)).toMatchObject({ phase: "finishing" }); + expect(await jobs.imports.abort(run.id, context)).toBe("marked"); + expect(await record(jobs.store, run.id)).toMatchObject({ + phase: "finishing", + abort: true, + outcome: { kind: "done", result: { count: 1 } }, + }); + await jobs.crash(); + }); +}); + +describe("abort racing an outcome", () => { + it("an abort that comes as run returns keeps its result: marked or terminal, the outcome done", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), async () => ({ count: 5 })); + const run = await jobs.imports.start({ path: "a" }, context); + const aborted = await jobs.imports.abort(run.id, context); + expect(["marked", "terminal"]).toContain(aborted); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 5 } }); + await jobs.close(); + }); +}); + +describe("pinning", () => { + it("an unsettled run's state never idles: past the residency retention it is in memory, read synchronously", async () => { + const open = await storeFile(); + const touched = vi.spyOn(Residency.prototype, "touch"); + const gate = Promise.withResolvers(); + const values: number[] = []; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + values.push(ctx.state.value.done); + return { count: ctx.state.value.done }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + vi.advanceTimersByTime(2 * RETENTION_MS); + } finally { + vi.useRealTimers(); + } + await delivered(); + // The run's documents never entered the idle structure the retention evicts from. + expect(touched).not.toHaveBeenCalled(); + expect(jobs.imports.running.value).toMatchObject([{ id: run.id, state: { done: 3 } }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ kind: "done", result: { count: 3 } }); + expect(values).toEqual([3]); + touched.mockRestore(); + await jobs.close(); + }); +}); + +describe("deferred state (D8)", () => { + it("is adopted at once, held, and stored with the outcome", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + let seen: number | undefined; + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { + persistence: { kind: "deferred", withinMs: Infinity }, + }, + ); + seen = ctx.state.value.done; + await gate.promise; + return { count: 1 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + expect(seen).toBe(1); + const state = await jobs.store.read( + (executor) => + executor.all( + "SELECT r.kind FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states'", + ), + context, + ); + expect(state).toEqual([{ kind: "base" }]); + gate.resolve(); + expect(await run.outcome(context)).toMatchObject({ state: { done: 1 } }); + await jobs.close(); + }); + + it("merges held ops into the next immediate change: one delta on the stored revision", async () => { + const open = await storeFile(); + const gate = Promise.withResolvers(); + const jobs = await opened(await open(), async (_args, ctx) => { + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + deferred, + ); + await ctx.state.change( + (draft) => { + draft.done = 2; + }, + ctx, + deferred, + ); + await ctx.state.change((draft) => { + draft.done = 3; + }, ctx); + await gate.promise; + return { count: 3 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + const revisions = await jobs.store.read( + (executor) => + executor.all<{ kind: string; content: string }>( + "SELECT r.kind, r.content FROM chord_revisions r JOIN chord_documents d ON d.id = r.document_id WHERE d.namespace = 'chord.job-states' ORDER BY r.seq", + ), + context, + ); + expect(revisions.map(({ kind, content }) => [kind, JSON.parse(content)])).toEqual([ + ["base", { done: 0 }], + [ + "delta", + // The held sets of one leaf folded (`PendingOps`), then the change's own. + [ + ["s", ["done"], 2], + ["s", ["done"], 3], + ], + ], + ]); + gate.resolve(); + await run.outcome(context); + await jobs.close(); + }); + + it("folds held writes: 100 sets of one leaf keep one op; past the pending bound the next change stores a base", async () => { + const open = await storeFile(); + const store = await open(); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + const jobs = await opened(store, async (_args, ctx) => { + for (let n = 1; n <= 100; n++) { + await ctx.state.change( + (draft) => { + draft.done = n; + }, + ctx, + deferred, + ); + } + await ctx.state.change((draft) => { + draft.done = 101; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + expect((await stateRevisions(store)).at(-1)).toEqual([ + "delta", + [ + ["s", ["done"], 100], + ["s", ["done"], 101], + ], + ]); + await jobs.close(); + }); + + it("an overflow of held ops stores the next change as a base", async () => { + const open = await storeFile(); + const store = await open(); + const Items = defineState("import.progress", { + schema: obj({ done: int(), items: arr(int()) }), + version: 1, + initial: () => ({ done: 0, items: new Array(MAX_PENDING_OPS + 2).fill(0) }), + persist: true, + checkpointWhen: () => false, + }); + const declared = defineJob("notes.import", { ...importOptions(), state: Items }); + const changed = Promise.withResolvers(); + const deferred = { persistence: { kind: "deferred", withinMs: Infinity } } as const; + // The declaration's state type differs from `Import`'s; this run writes only `Items`' shape. + const jobs = await opened( + store, + async (_args, ctx) => { + const state = ctx.state as unknown as RunState<{ done: number; items: number[] }>; + // One leaf each: nothing folds, and the queue passes its bound. + for (let n = 0; n <= MAX_PENDING_OPS; n++) { + await state.change( + (draft) => { + draft.items[n] = 1; + }, + ctx, + deferred, + ); + } + await state.change((draft) => { + draft.done = 1; + }, ctx); + changed.resolve(); + return new Promise(() => {}); + }, + undefined, + declared as unknown as typeof Import, + ); + await jobs.imports.start({ path: "a" }, context); + await changed.promise; + // A new base prunes the older revisions (`history: none`): the start's base is gone. + expect(await stateRevisions(store)).toEqual([["base", expect.objectContaining({ done: 1 })]]); + await jobs.close(); + }); + + it("is stored by the close writer at a clean stop, and after withinMs by the object's timer", async () => { + const open = await storeFile(); + let jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 4; + }, + ctx, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + return new Promise(() => {}); + }); + await jobs.imports.start({ path: "a" }, context); + await delivered(); + await delivered(); + await jobs.close(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 4 }); + await ctx.state.change( + (draft) => { + draft.done = 5; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 10 } }, + ); + return new Promise(() => {}); + }); + await new Promise((resolve) => setTimeout(resolve, 60)); + await jobs.crash(); + jobs = await opened(await open(), async (_args, ctx) => { + expect(ctx.state.value).toEqual({ done: 5 }); + return { count: 0 }; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(jobs.finished).toMatchObject([{ kind: "done", state: { done: 5 } }]); + await jobs.close(); + }); + + it("refuses a negative withinMs", async () => { + const open = await storeFile(); + let refused: unknown; + const jobs = await opened(await open(), async (_args, ctx) => { + refused = await ctx.state + .change(() => {}, ctx, { persistence: { kind: "deferred", withinMs: -1 } }) + .catch((error: unknown) => error); + return { count: 0 }; + }); + const run = await jobs.imports.start({ path: "a" }, context); + await run.outcome(context); + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await jobs.close(); + }); +}); + +describe("GC", () => { + const runs = instancesOf(JobRun, "entryFor"); + + it("settled runs, cancelled abort waits, and an ended object with a held change and its timer are collected", async () => { + const open = await storeFile(); + /** Runs one settled run and leaves one held change behind a closed host: what must be collected. */ + const ended = async (): Promise[]> => { + // Earlier tests leave crashed hosts behind: only this one's objects and runs count. + runs.splice(0); + attempted.splice(0); + const jobs = await opened(await open(), async (_args, ctx) => { + await ctx.state.change( + (draft) => { + draft.done = 1; + }, + ctx, + { persistence: { kind: "deferred", withinMs: 60_000 } }, + ); + return new Promise(() => {}); + }); + const settled = await jobs.abandoned.start({ path: "a" }, context); + await settled.outcome(context); + await jobs.imports.start({ path: "b" }, context); + await delivered(); + await delivered(); + const refs = [...runs.splice(0), ...attempted.splice(0)]; + await jobs.close(); + return refs; + }; + await expectCollected(...(await ended())); + const hanging = await opened(await open(), () => new Promise(() => {})); + const run = await hanging.imports.start({ path: "c" }, context); + await delivered(); + const cancel = withCancel(context); + let waiting: Promise | undefined = hanging.imports.abort(run.id, cancel.context).catch(() => {}); + const waitRef = new WeakRef(waiting); + await delivered(); + cancel.cancel(); + await waiting; + waiting = undefined; + await expectCollected(waitRef); + await hanging.close(); + }); +}); diff --git a/packages/chord/test/next/jobs/records.test.ts b/packages/chord/test/next/jobs/records.test.ts index 736791615..d8d2fca8b 100644 --- a/packages/chord/test/next/jobs/records.test.ts +++ b/packages/chord/test/next/jobs/records.test.ts @@ -114,7 +114,7 @@ describe("runs across attempts", () => { }); const run = await jobs[0]!.start({ text: "a" }, context); await flushed(); - jobs[0]!.abort(run.id); + void jobs[0]!.abort(run.id, context); await flushed(); expect(attempts).toEqual([1]); await restart(); @@ -176,7 +176,7 @@ describe("the object's end during an admitted commit", () => { apply: async () => { inStep.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); diff --git a/packages/chord/test/next/jobs/run.test.ts b/packages/chord/test/next/jobs/run.test.ts index 0899a33f7..1c2bb89a1 100644 --- a/packages/chord/test/next/jobs/run.test.ts +++ b/packages/chord/test/next/jobs/run.test.ts @@ -7,8 +7,8 @@ import { JobRun } from "../../../src/next/jobs/run.ts"; import { rejection } from "../support.ts"; describe("a run", () => { - it("has an entry per phase: args and state while running or finishing, neither when blocked", () => { - const run = new JobRun("r1", 2, "k", { command: "ls" }); + it("has an entry per phase: args and state while running or finishing, neither when blocked, with its reason", () => { + const run = new JobRun("r1", 2, "k", { command: "ls" }, "running"); expect(run.entryFor({ output: "" })).toEqual({ id: "r1", version: 2, @@ -18,23 +18,31 @@ describe("a run", () => { phase: "running", }); run.phase = "finishing"; - expect(new JobRun("r2", 1, undefined, { a: 1 }).entryFor(undefined)).toEqual({ + expect(new JobRun("r2", 1, undefined, { a: 1 }, "running").entryFor(undefined)).toEqual({ id: "r2", version: 1, args: { a: 1 }, phase: "running", }); run.phase = "blocked"; + run.blocked = "finish_failed"; expect(run.entryFor({ output: "x" })).toEqual({ id: "r1", version: 2, phase: "blocked", reason: "finish_failed", }); + run.blocked = "version"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "blocked", reason: "version" }); + // A persisted run waiting for its reservation, or stopped, runs (again): `running`. + run.phase = "pending"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); + run.phase = "stopped"; + expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); }); it("gives every waiter the outcome, late ones too; a waiter's context ends only its wait", async () => { - const run = new JobRun("r1", 1, undefined, {}); + const run = new JobRun("r1", 1, undefined, {}, "running"); const { context: waiting, cancel } = withCancel(context); const cancelled = rejection(run.handle.outcome(waiting)); cancel(new Error("left")); @@ -42,7 +50,7 @@ describe("a run", () => { const outcome = { id: "r1", args: {}, state: undefined, kind: "done", result: 1 } as const; run.resolve(outcome); expect(await run.handle.outcome(context)).toBe(outcome); - const rejected = new JobRun("r2", 1, undefined, {}); + const rejected = new JobRun("r2", 1, undefined, {}, "running"); rejected.reject(new ChordError({ code: "aborted", message: "dropped" })); expect(await rejection(rejected.handle.outcome(context))).toMatchObject({ code: "aborted" }); }); diff --git a/packages/chord/test/next/state/watchable.test.ts b/packages/chord/test/next/state/watchable.test.ts index f59f113c1..35184f31f 100644 --- a/packages/chord/test/next/state/watchable.test.ts +++ b/packages/chord/test/next/state/watchable.test.ts @@ -119,6 +119,7 @@ describe("subscribers", () => { const subscriber: Subscriber = { advance: (call) => void (call.kind === "value" && frames.push([call.value, call.ops])), end: () => {}, + endAfterDelivery: () => {}, }; source.subscribe(subscriber); expect(source.watched).toBe(true); diff --git a/packages/chord/test/next/store/allocator.test.ts b/packages/chord/test/next/store/allocator.test.ts new file mode 100644 index 000000000..518787f6d --- /dev/null +++ b/packages/chord/test/next/store/allocator.test.ts @@ -0,0 +1,24 @@ +// Tests of the id allocator (slice 6b D3, §1.2): from 2, raised by explicit ids, saturating at the safe range. + +import { describe, expect, it } from "vitest"; +import { FIRST_ID, IdAllocator } from "../../../src/next/store/allocator.ts"; + +describe("the allocator", () => { + it("mints from its value, and an explicit id raises it above itself, never lowering it", () => { + const ids = new IdAllocator(FIRST_ID); + expect([ids.mint(), ids.mint()]).toEqual([2, 3]); + ids.raise(100); + expect(ids.mint()).toBe(101); + ids.raise(50); + expect(ids.value).toBe(102); + }); + + it("saturates at Number.MAX_SAFE_INTEGER: that id may be written, every later mint is refused", () => { + const ids = new IdAllocator(Number.MAX_SAFE_INTEGER - 1); + expect(ids.mint()).toBe(Number.MAX_SAFE_INTEGER - 1); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + ids.raise(Number.MAX_SAFE_INTEGER); + expect(ids.value).toBe(Number.MAX_SAFE_INTEGER); + expect(() => ids.mint()).toThrow("ID space is exhausted"); + }); +}); diff --git a/packages/chord/test/next/store/applier.test.ts b/packages/chord/test/next/store/applier.test.ts index 00d8e61f5..5cad8bfe8 100644 --- a/packages/chord/test/next/store/applier.test.ts +++ b/packages/chord/test/next/store/applier.test.ts @@ -3,14 +3,16 @@ // as stored, booleans, json(...) cells as JSON text) and documents with their ids kept. import { describe, expect, it } from "vitest"; -import { type AppliedWrite, applyWrite } from "../../../src/next/store/applier.ts"; +import { IdAllocator } from "../../../src/next/store/allocator.ts"; +import { applyWrite } from "../../../src/next/store/applier.ts"; import { Step } from "../../../src/next/store/step.ts"; +import type { StoreWrite } from "../../../src/next/store/writes.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { rejection } from "../support.ts"; -const TABLE = "client:app.store:edits"; +const TABLE = "app.store:edits"; -const table: AppliedWrite = { +const table: StoreWrite = { kind: "table", table: TABLE, version: 2, @@ -26,11 +28,11 @@ const table: AppliedWrite = { }; /** Applies `writes` in one transaction of a fresh database; resolves with the rows of the table. */ -async function applied(...writes: AppliedWrite[]): Promise { +async function applied(...writes: StoreWrite[]): Promise { const db = await openNodeSqliteDatabase(":memory:"); try { return await db.transaction(async (executor) => { - const step = new Step(executor, 1, 1, new Map()); + const step = new Step(executor, 1, new IdAllocator(2), new Map()); for (const write of writes) await applyWrite(step, write, "test"); return executor.all(`SELECT * FROM "${TABLE}"`); }); @@ -51,7 +53,7 @@ describe("applied writes", () => { expect(rows).toEqual([{ id: "a", done: 1, note: null, meta: '{"x":[1]}', meta$v: 1, $v: 1 }]); }); - it.each<[string, AppliedWrite[], string]>([ + it.each<[string, StoreWrite[], string]>([ [ "a put for a table not recorded", [{ kind: "put", table: TABLE, row: {}, version: 1, columns: {} }], @@ -59,17 +61,17 @@ describe("applied writes", () => { ], [ "a key over an opt column", - [{ ...table, key: ["note"] } as AppliedWrite], + [{ ...table, key: ["note"] } as StoreWrite], '"note" is not a non-opt str or int column', ], [ "a json column without a fingerprint", - [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as AppliedWrite], + [{ ...table, columns: { id: { type: "str" }, meta: { type: "json", version: 1 } } } as StoreWrite], "a json(...) column has a version and a fingerprint", ], [ "a default of another type", - [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as AppliedWrite], + [{ ...table, columns: { id: { type: "str" }, n: { type: "int", default: "x" } } } as StoreWrite], "is not a int value", ], [ diff --git a/packages/chord/test/next/store/apply-batch.test.ts b/packages/chord/test/next/store/apply-batch.test.ts new file mode 100644 index 000000000..762b4f4d6 --- /dev/null +++ b/packages/chord/test/next/store/apply-batch.test.ts @@ -0,0 +1,290 @@ +// Tests of batch apply (slice 6b §1.6, S5, S17): a store's outbox replayed into an empty store gives the same archive, +// manifest included; every write kind (an adapter's create-plus-retire and change-plus-retire too); the refusals (claims held, seqs that do not follow, writes the records refuse) +// and the divergence of a recorded count. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; +import { applyBatch } from "../../../src/next/store/apply-batch.ts"; +import { + changeDocument, + createDocument, + type DocumentRecord, + type DocumentWrite, + insertDocument, + readDocument, + retireDocument, + retireRecord, + writeContent, +} from "../../../src/next/store/engine.ts"; +import { add, type ColumnRef, lt, prefix } from "../../../src/next/store/expression.ts"; +import type { OutboxCommit } from "../../../src/next/store/outbox.ts"; +import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { ChordStore } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { prepareTables } from "../../../src/next/store/table-migration.ts"; +import { archiveStore, exported, memoryStore, rejection, tableIn } from "../support.ts"; + +const OPEN = { admit: () => true, adopt: () => {} }; + +/** The column references a table's callbacks get (the changes are untyped below the transaction). */ +type Refs = { readonly [column: string]: ColumnRef | undefined }; + +const CountsV1 = defineTable("counts", { + version: 1, + columns: { id: str(), n: int(), tag: str() }, + key: ["id"], + indexes: { by_tag: ["tag"] }, +}); + +const CountsV2 = evolveTable(CountsV1, (t) => t.add("extra", int({ default: 0 })).dropIndex("by_tag")); + +/** A store with an outbox from its first commit holding every write kind: an archive store, then counted writes. */ +async function everyKind(): Promise { + const store = await archiveStore({ outbox: true }); + const counts = await tableIn(store, CountsV1); + const staged: StagedWrite[] = []; + const changes = new TableChanges(counts, staged); + for (let i = 0; i < 10; i++) changes.put({ id: `c${i}`, n: i, tag: i % 2 === 0 ? "even" : "odd" }); + await store.commit(staged, OPEN, context); + const bulk: StagedWrite[] = []; + const more = new TableChanges(counts, bulk); + more.update({ id: "c1" }, (c: Refs) => ({ n: add(c.n!, 10) })); + more.delete({ id: "c2" }); + more.updateWhere((c: Refs) => prefix(c.tag!, "ev"), { tag: "EVEN" }); + more.deleteWhere((c: Refs) => lt(c.n!, 1)); + await store.commit(bulk, OPEN, context); + await prepareTables([{ definition: CountsV2, table: "app.store:counts" }], store, OPEN, "app.store", context); + const created: number[] = []; + const fence = { admit: () => true, adopt: (_: number, ids: readonly number[]) => void created.push(...ids) }; + const address = { owner: "app.store", namespace: "chord", kind: "replayed", key: "x" }; + const base = { version: 1, fingerprint: "f1", value: { n: 0 } }; + await store.commit([createDocument(address, base, '{"n":0}')], fence, context); + const ops = [["s", ["n"], 1]] as const; + await store.commit( + [changeDocument(created[0]!, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], + OPEN, + context, + ); + await store.commit([retireDocument(created[0]!, 2)], OPEN, context); + await durableShaped(store); + return store; +} + +/** + * What only an adapter writes (Durable's commits): a rewindable creation retired in its own commit (an empty + * lifetime), and a change with the retirement of the same record in one commit. + */ +async function durableShaped(store: ChordStore): Promise { + const address = { owner: "app.durable", namespace: "session", kind: "notes" }; + const base = { version: 1, fingerprint: "", value: { n: 0 } }; + /** A record created at `seq`, retired then too when `retired`. */ + const made = (id: number, key: string, seq: number, retired: boolean): DocumentRecord => ({ + id, + ...address, + key, + history: "rewindable", + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: "{}", + revision: 1, + version: 1, + fingerprint: "", + }); + const create = (key: string, retired: boolean): StagedWrite => ({ + order: "write", + apply: async (step) => { + const record = made(step.ids.mint(), key, step.seq, retired); + await insertDocument(step, record, '{"n":0}'); + return [{ kind: "document.create", record, base }]; + }, + }); + await store.commit([create("empty", true), create("kept", false)], OPEN, context); + const kept = (await store.read( + (executor) => executor.get<{ id: number }>("SELECT id FROM chord_documents WHERE key = 'kept'"), + context, + ))!.id; + const ops = [["s", ["n"], 1]] as const; + const changeAndRetire: StagedWrite = { + order: "write", + apply: async (step) => { + const record = (await readDocument(step, kept))!; + const content = { kind: "delta", version: 1, ops } as const; + await writeContent(step, kept, "rewindable", 1, content, JSON.stringify(ops)); + await retireRecord(step, record); + const writes: DocumentWrite[] = [ + { kind: "document.change", id: kept, revision: 1, content }, + { kind: "document.retire", id: kept, revision: 2 }, + ]; + return writes; + }, + }; + await store.commit([changeAndRetire], OPEN, context); +} + +/** Every commit of `store`'s outbox. */ +async function commits(store: ChordStore): Promise { + return store.outbox!.read(store.outbox!.floor, 10_000, context); +} + +describe("replay", () => { + it("a store's outbox replayed into an empty store gives the same archive, manifest included", async () => { + const source = await everyKind(); + const all = await commits(source); + const kinds = new Set(all.flatMap(({ writes }) => writes.map(({ kind }) => kind))); + expect([...kinds].sort()).toEqual( + [ + "alter", + "document.change", + "document.create", + "document.retire", + "dropIndex", + "index", + "put", + "table", + "update", + "updateWhere", + "delete", + "deleteWhere", + ].sort(), + ); + const replica = await memoryStore(); + await applyBatch(replica, all.slice(0, 3), context); + await applyBatch(replica, all.slice(3), context); + expect(await exported(replica)).toEqual(await exported(source)); + expect([replica.lastSeq, replica.nextId]).toEqual([source.lastSeq, source.nextId]); + }); + + it("restores the allocator: a replica mints above every id it holds", async () => { + const source = await everyKind(); + const replica = await memoryStore(); + await applyBatch(replica, await commits(source), context); + expect(replica.mintId()).toBe(source.nextId); + }); + + it("writes the applied commits to the replica's own outbox when it has one", async () => { + const source = await everyKind(); + const all = await commits(source); + const replica = await memoryStore({ outbox: true }); + await applyBatch(replica, all, context); + expect(await commits(replica)).toEqual(all); + }); +}); + +describe("refusals", () => { + it("while an owner holds a claim on the store (its memory would go stale)", async () => { + const replica = await memoryStore(); + replica.claims.documents("app.store", "chord", { label: "an object" }); + expect(await rejection(applyBatch(replica, [], context))).toMatchObject({ code: "invalid", reason: "args" }); + }); + + it("a claim taken between the call and its turn in the queue refuses the batch; a claim is refused while it applies", async () => { + const replica = await memoryStore(); + const held = await replica.take(() => {}); + const replaying = applyBatch(replica, [], context); + const owner = { label: "an object" }; + replica.claims.documents("app.store", "chord", owner); + held(); + expect(await rejection(replaying)).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + replica.claims.release(owner); + const all = await commits(await everyKind()); + let applied = false; + const applying = applyBatch(replica, all, context).then(() => { + applied = true; + }); + // A claim made and released in one turn holds nothing; once the batch applies (across awaits), one is refused. + let refused: unknown; + while (!applied && refused === undefined) { + await Promise.resolve(); + try { + replica.claims.documents("app.store", "chord", owner); + replica.claims.release(owner); + } catch (error) { + refused = error; + } + } + expect(refused).toMatchObject({ code: "invalid", reason: "args" }); + await applying; + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + replica.claims.documents("app.store", "chord", owner); + }); + + it("a batch queued behind one that diverges is refused at its turn", async () => { + const all = await commits(await everyKind()); + const diverging = structuredClone(all); + for (const commit of diverging) { + for (const write of commit.writes) if (write.kind === "updateWhere") (write as { count: number }).count += 1; + } + const replica = await memoryStore(); + const first = rejection(applyBatch(replica, diverging, context)); + const second = rejection(applyBatch(replica, all, context)); + expect(String(await first)).toContain("recorded"); + expect(await second).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); + + it("a commit whose next id is not above a document it creates (a saturated id aside)", async () => { + const replica = await memoryStore(); + const record = { + id: 100, + owner: "app.store", + namespace: "chord", + kind: "x", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const base = { version: 1, fingerprint: "", value: {} }; + const create = { kind: "document.create", record, base } as const; + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 2, writes: [create] }], context))).toMatchObject({ + code: "invalid", + reason: "args", + }); + expect(await rejection(applyBatch(replica, [{ seq: 1, nextId: 100, writes: [create] }], context))).toMatchObject({ + reason: "args", + }); + await applyBatch(replica, [{ seq: 1, nextId: 101, writes: [create] }], context); + const saturated = { ...create, record: { ...record, id: Number.MAX_SAFE_INTEGER, key: "max" } }; + await applyBatch(replica, [{ seq: 2, nextId: Number.MAX_SAFE_INTEGER, writes: [saturated] }], context); + expect(replica.lastSeq).toBe(2); + }); + + it("a first commit that does not follow lastSeq, or one that does not follow the one before; nothing applied", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + expect(await rejection(applyBatch(replica, all.slice(1), context))).toMatchObject({ reason: "args" }); + expect(await rejection(applyBatch(replica, [all[0]!, all[2]!], context))).toMatchObject({ reason: "args" }); + expect(replica.lastSeq).toBe(0); + await applyBatch(replica, all, context); + expect(replica.lastSeq).toBe(all.at(-1)!.seq); + }); + + it("a write the records refuse rolls the batch back, and the store goes on", async () => { + const all = await commits(await everyKind()); + const replica = await memoryStore(); + const broken = structuredClone(all); + (broken[0]!.writes[0] as { table: string }).table = "no:such:table"; + expect(await rejection(applyBatch(replica, broken, context))).toMatchObject({ code: "invalid", reason: "args" }); + const forged = structuredClone(all); + (forged[0]!.writes as unknown[]).push({ kind: "put", table: "x", row: {}, version: 0 }); + expect(await rejection(applyBatch(replica, forged, context))).toMatchObject({ code: "invalid", reason: "args" }); + await applyBatch(replica, all, context); + }); + + it("a recorded count the store does not reproduce stops applying and marks the store diverged (S17)", async () => { + const all = await commits(await everyKind()); + const index = all.findIndex(({ writes }) => writes.some(({ kind }) => kind === "updateWhere")); + const diverging = structuredClone(all); + const write = diverging[index]!.writes.find(({ kind }) => kind === "updateWhere") as { count: number }; + write.count += 1; + const replica = await memoryStore(); + expect(String(await rejection(applyBatch(replica, diverging, context)))).toContain("recorded"); + expect(await rejection(applyBatch(replica, all, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(replica.lastSeq).toBe(0); + }); +}); diff --git a/packages/chord/test/next/store/catalog.test.ts b/packages/chord/test/next/store/catalog.test.ts index 451c45cb2..11761b30a 100644 --- a/packages/chord/test/next/store/catalog.test.ts +++ b/packages/chord/test/next/store/catalog.test.ts @@ -20,7 +20,7 @@ const Edits = defineTable("edits", { key: ["id"], indexes: { by_at: ["at"] }, }); -const TABLE = "session:app.store:edits"; +const TABLE = "app.store:edits"; describe("the catalog", () => { it("is read once at open: a reopened store knows its tables", async () => { @@ -51,14 +51,12 @@ describe("the catalog", () => { apply: (step) => step .run("INSERT INTO no_such_table VALUES (1)") - .then(() => ({ kind: "dropIndex", table: TABLE, name: "x" })), + .then(() => [{ kind: "dropIndex", table: TABLE, name: "x" }]), }; const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); - await expect( - store.commit([table, failing], { admit: () => true, adopt: () => {} }, context), - ).rejects.toMatchObject({ - reason: "storage_failed", - }); + await expect(store.commit([table, failing], { admit: () => true, adopt: () => {} }, context)).rejects.toThrow( + "no such table", + ); expect(store.table(TABLE)).toBeUndefined(); await store.close(context); }); diff --git a/packages/chord/test/next/store/claims.test.ts b/packages/chord/test/next/store/claims.test.ts new file mode 100644 index 000000000..3974487e3 --- /dev/null +++ b/packages/chord/test/next/store/claims.test.ts @@ -0,0 +1,72 @@ +// Tests of claims (slice 6b "owner names and claims", "claims per table and per document namespace"): a table by its +// SQL name and documents by owner and namespace, once per store; released by their claimant's end; none taken while a +// batch applies. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { Claims } from "../../../src/next/store/claims.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore } from "../support.ts"; + +describe("claims", () => { + it("refuse a second claimant of a table or of an owner's namespace; one claimant may claim again", () => { + const claims = new Claims(); + const a = { label: "A" }; + const b = { label: "B" }; + claims.table("x:edits", a); + claims.table("x:edits", a); + claims.documents("x", "chord", a); + expect(() => claims.table("x:edits", b)).toThrow("claimed by A"); + expect(() => claims.documents("x", "chord", b)).toThrow("claimed by A"); + // Another namespace of the same owner, another table: no collision. + claims.documents("x", "session", b); + claims.table("x:entries", b); + expect(claims.held).toBe(true); + claims.release(a); + claims.table("x:edits", b); + claims.release(b); + expect(claims.held).toBe(false); + }); + + it("are refused while locked (a batch applies), taken again once unlocked", () => { + const claims = new Claims(); + const owner = { label: "A" }; + const unlock = claims.lock(); + expect(() => claims.table("x:edits", owner)).toThrow("while a batch applies"); + expect(() => claims.documents("x", "chord", owner)).toThrow("while a batch applies"); + expect(claims.held).toBe(false); + unlock(); + claims.table("x:edits", owner); + expect(claims.held).toBe(true); + }); + + it("an object claims its tables and documents when it declares them, and releases them at its end", async () => { + const store = await memoryStore(); + const Note = defineState("note", { + schema: obj({ n: int() }), + version: 1, + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => true, + }); + const Edits = defineTable("edits", { version: 1, columns: { id: str() }, key: ["id"] }); + const Notes = local("notes.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded( + (h) => + h.serve(Notes, (_, self) => { + self.state(Note); + self.table(Edits); + return {}; + }), + { store }, + ); + await host.get(Notes, {}, context); + expect(() => store.claims.table("notes.store:edits", { label: "other" })).toThrow("claimed by notes.store"); + expect(() => store.claims.documents("notes.store", "chord", { label: "other" })).toThrow("claimed"); + await host.close(context); + expect(store.claims.held).toBe(false); + store.claims.table("notes.store:edits", { label: "other" }); + }); +}); diff --git a/packages/chord/test/next/store/engine.test.ts b/packages/chord/test/next/store/engine.test.ts index 78a41dab0..00b28eae4 100644 --- a/packages/chord/test/next/store/engine.test.ts +++ b/packages/chord/test/next/store/engine.test.ts @@ -1,4 +1,5 @@ -// The engine (slice 6a §10, §11): persisted documents as bases and deltas, their keys in code point order, retirement. +// The engine (slice 6a §10, §11, slice 6b §1.5): documents as bases and deltas, their keys in code point order, the +// history rules at a new base and at retirement (R1), reads at a point, records inserted as they are. import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; @@ -9,10 +10,17 @@ import { changeDocument, createDocument, type DocumentAddress, + type DocumentHistory, + type DocumentRecord, findDocument, + insertDocument, listDocumentKeys, + listDocuments, loadDocument, + readDocument, retireDocument, + retireRecord, + writeContent, } from "../../../src/next/store/engine.ts"; import { importStore } from "../../../src/next/store/import.ts"; import { TABLES } from "../../../src/next/store/schema.ts"; @@ -21,7 +29,7 @@ import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { exported, rejection } from "../support.ts"; -const OWNER = { namespace: "chord", owner: "session:app.store", kind: "notes" }; +const OWNER = { owner: "app.store", namespace: "chord", kind: "notes" }; /** A store on `:memory:` (or `on`) and a commit helper resolving with the created ids. */ async function engine(on?: ChordStore) { @@ -47,7 +55,7 @@ async function engine(on?: ChordStore) { fingerprint: `f${version}`.padEnd(16, "0"), value: value as never, }); - const create = (key: string, value: object) => createDocument(at(key), "latest", base(value), JSON.stringify(value)); + const create = (key: string, value: object) => createDocument(at(key), base(value), JSON.stringify(value)); const delta = (id: number, revision: number, ops: Op[]) => changeDocument(id, revision, { kind: "delta", version: 1, ops }, JSON.stringify(ops)); const read = (fn: Parameters[0]) => store.read(fn, context) as Promise; @@ -60,6 +68,16 @@ async function reimported(store: ChordStore, edit: (parts: ArchivePart[]) => unk return (await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context)) as ChordStore; } +/** A staged write applying `apply` in the step (an adapter's composite). */ +function staged(apply: StagedWrite["apply"]): StagedWrite { + return { order: "write", apply }; +} + +/** An adapter's record of `history` at `key`, with an explicit id and the step's seq. */ +function record(id: number, key: string, history: DocumentHistory, createdAt: number): DocumentRecord { + return { id, ...OWNER, key, history, createdAt, revision: 1, version: 1, fingerprint: "" }; +} + const LIMIT = Number.MAX_SAFE_INTEGER; describe("documents", () => { @@ -68,12 +86,12 @@ describe("documents", () => { const [id] = await commit(create("a", { n: 0 })); await commit(delta(id!, 1, [["s", ["n"], 1]])); await commit(delta(id!, 2, [["s", ["n"], 2]])); - const record = (await read>>((e) => findDocument(e, at("a"))))!; - expect(record).toMatchObject({ id, revision: 3, version: 1, history: "latest", createdAt: 1 }); - expect(await read((e) => loadDocument(e, record))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); + const found = (await read((e) => findDocument(e, at("a"))))!; + expect(found).toMatchObject({ id, revision: 3, version: 1, history: "none", createdAt: 1 }); + expect(await read((e) => loadDocument(e, found))).toMatchObject({ value: { n: 2 }, deltasSinceBase: 2 }); const next = base({ m: 9 }, 2); await commit(changeDocument(id!, 3, { kind: "base", ...next }, JSON.stringify(next.value))); - const rebased = (await read>>((e) => findDocument(e, at("a"))))!; + const rebased = (await read((e) => findDocument(e, at("a"))))!; expect(rebased).toMatchObject({ revision: 4, version: 2, fingerprint: next.fingerprint }); expect(await read((e) => loadDocument(e, rebased))).toMatchObject({ value: { m: 9 }, deltasSinceBase: 0 }); expect(await read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([{ seq: 4 }]); @@ -86,45 +104,152 @@ describe("documents", () => { expect(await read((e) => listDocumentKeys(e, OWNER, "b", 1))).toEqual(["\uFF5E"]); }); - it("retire a latest record with its revisions (rule 19); the address takes a new incarnation", async () => { + it("retire a none record with its row and revisions (R1); the address takes a new incarnation", async () => { const { commit, at, create, read } = await engine(); const [id] = await commit(create("a", { n: 0 })); const [next] = await commit(retireDocument(id!, 1), create("a", { n: 1 })); - expect(next).toBe(2); - expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 2 }]); - expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 2 }]); - expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 2 }); + expect(next).toBe(3); + expect(await read((e) => e.all(`SELECT id FROM ${TABLES.documents}`))).toEqual([{ id: 3 }]); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions}`))).toEqual([{ document_id: 3 }]); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: 3 }); }); - it("refuse two live records at one address and a delta on another version: storage_failed", async () => { + it("refuse two live records at one address and a delta on another version: the step is refused", async () => { const first = await engine(); await first.commit(first.create("a", {})); - expect(await rejection(first.commit(first.create("a", {})))).toMatchObject({ reason: "storage_failed" }); + expect(String(await rejection(first.commit(first.create("a", {}))))).toContain("UNIQUE"); const second = await engine(); const [id] = await second.commit(second.create("a", {})); const crossing = changeDocument(id!, 1, { kind: "delta", version: 2, ops: [] }, "[]"); - expect(await rejection(second.commit(crossing))).toMatchObject({ reason: "storage_failed" }); + expect(String(await rejection(second.commit(crossing)))).toContain("not live at revision 1"); + expect(await second.read(async () => 1)).toBe(1); + }); +}); + +describe("history rules (R1) and reads at a point", () => { + /** Creates adapter record `id` at "a" (its base `{ n: 0 }`), then changes and retires it in later commits. */ + async function lifetime(history: DocumentHistory) { + const { store, commit, at, read } = await engine(); + await commit( + staged(async (step) => { + const made = record(10, "a", history, step.seq); + await insertDocument(step, made, '{"n":0}'); + return []; + }), + ); + await commit( + staged(async (step) => { + const content = { kind: "base", version: 1, fingerprint: "", value: { n: 1 } } as const; + await writeContent(step, 10, history, 1, content, '{"n":1}'); + return []; + }), + ); + await commit( + staged(async (step) => { + await writeContent( + step, + 10, + history, + 2, + { kind: "delta", version: 1, ops: [["s", ["n"], 2]] }, + '[["s",["n"],2]]', + ); + return []; + }), + ); + // An adapter's retirement (Durable's): the record read in the step, retired by its history's rule. + await commit( + staged(async (step) => { + await retireRecord(step, (await readDocument(step, 10))!); + return []; + }), + ); + return { store, read, at }; + } + + it("keep a retired latest record's row without revisions, and every revision of a rewindable one", async () => { + const latest = await lifetime("latest"); + expect(await latest.read((e) => e.all(`SELECT id, retired_at FROM ${TABLES.documents}`))).toEqual([ + { id: 10, retired_at: 4 }, + ]); + expect(await latest.read((e) => e.all(`SELECT seq FROM ${TABLES.revisions}`))).toEqual([]); + const rewindable = await lifetime("rewindable"); + expect(await rewindable.read((e) => e.all(`SELECT seq, kind FROM ${TABLES.revisions} ORDER BY seq`))).toEqual([ + { seq: 1, kind: "base" }, + { seq: 2, kind: "base" }, + { seq: 3, kind: "delta" }, + ]); + }); + + it("find a record at a point by its half-open lifetime, and load a rewindable one at any point inside it", async () => { + const { read, at } = await lifetime("rewindable"); + const found = await Promise.all( + [0, 1, 3, 4, "current" as const].map((point) => + read((e) => findDocument(e, at("a"), point)), + ), + ); + expect(found.map((each) => each?.id)).toEqual([undefined, 10, 10, undefined, undefined]); + const record = found[1]!; + expect(record.retiredAt).toBe(4); + for (const [point, n] of [ + [1, 0], + [2, 1], + [3, 2], + ] as const) { + expect(await read((e) => loadDocument(e, record, point))).toMatchObject({ value: { n }, version: 1 }); + } + expect(await read((e) => listDocuments(e, OWNER, 3, 0, 10))).toMatchObject([{ id: 10 }]); + expect(await read((e) => listDocuments(e, OWNER, "current", 0, 10))).toEqual([]); + }); + + it("find a retired latest record at a point, whose content is gone", async () => { + const { read, at } = await lifetime("latest"); + const found = (await read((e) => findDocument(e, at("a"), 2)))!; + expect(found).toMatchObject({ id: 10, retiredAt: 4, history: "latest" }); + // A read returns data (the store stops on a read's throw, slice 6b §1.4): the outcome as a value. + const outcome = read((e) => + loadDocument(e, found, 2).then( + () => "loaded", + (error: unknown) => String(error), + ), + ); + expect(await outcome).toContain("no base"); + }); + + it("insert a create-plus-retire as an empty lifetime that never holds the live address", async () => { + const { commit, read, at, create } = await engine(); + const [live] = await commit(create("a", {})); + await commit( + staged(async (step) => { + await insertDocument(step, { ...record(20, "a", "latest", step.seq), retiredAt: step.seq }, "{}"); + return []; + }), + ); + expect(await read((e) => findDocument(e, at("a")))).toMatchObject({ id: live }); + expect(await read((e) => e.all(`SELECT document_id FROM ${TABLES.revisions} WHERE document_id = 20`))).toEqual( + [], + ); }); }); describe("counters at the safe-integer limit", () => { - it("allocate document ids up to the safe range after an import near it; past it a commit fails storage_failed", async () => { + it("allocate document ids up to the safe range after an import near it; past it the commit is refused", async () => { const { store: source } = await engine(); const near = (next: number) => - reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextDocumentId: next }, ...rest]); + reimported(source, ([manifest, ...rest]) => [{ ...manifest, nextId: next }, ...rest]); const { store, commit, create } = await engine(await near(LIMIT - 1)); expect(await commit(create("a", { n: 0 }))).toEqual([LIMIT - 1]); - expect(store.nextDocumentId).toBe(LIMIT); - expect(await rejection(commit(create("b", { n: 0 })))).toMatchObject({ reason: "storage_failed" }); + expect(store.nextId).toBe(LIMIT); + expect(String(await rejection(commit(create("b", { n: 0 }))))).toContain("ID space is exhausted"); // Two creations in one commit where only one id is left: nothing of it is written. const both = await engine(await near(LIMIT - 1)); - expect(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 })))).toMatchObject({ - reason: "storage_failed", - }); - expect([both.store.lastSeq, both.store.nextDocumentId]).toEqual([0, LIMIT - 1]); + expect(String(await rejection(both.commit(both.create("a", { n: 0 }), both.create("b", { n: 0 }))))).toContain( + "ID space is exhausted", + ); + expect([both.store.lastSeq, both.store.nextId]).toEqual([0, LIMIT - 1]); }); - it("count revisions up to the safe range after an import near it; a change past it fails storage_failed", async () => { + it("count revisions up to the safe range after an import near it; a change past it is refused", async () => { const { store: source, commit: seed, create } = await engine(); await seed(create("a", { n: 0 })); const store = await reimported(source, (parts) => @@ -133,10 +258,8 @@ describe("counters at the safe-integer limit", () => { ), ); const { commit, delta, read, at } = await engine(store); - const record = (await read>>((e) => findDocument(e, at("a"))))!; - await commit(delta(record.id, LIMIT - 1, [["s", ["n"], 1]])); - expect(await rejection(commit(delta(record.id, LIMIT, [["s", ["n"], 2]])))).toMatchObject({ - reason: "storage_failed", - }); + const found = (await read((e) => findDocument(e, at("a"))))!; + await commit(delta(found.id, LIMIT - 1, [["s", ["n"], 1]])); + expect(String(await rejection(commit(delta(found.id, LIMIT, [["s", ["n"], 2]]))))).toContain("safe range"); }); }); diff --git a/packages/chord/test/next/store/export.test.ts b/packages/chord/test/next/store/export.test.ts index e0e743023..9a8d70076 100644 --- a/packages/chord/test/next/store/export.test.ts +++ b/packages/chord/test/next/store/export.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; import { int, str } from "../../../src/next/contracts/index.ts"; import { ARCHIVE_FORMAT } from "../../../src/next/store/archive.ts"; +import { insertDocument, insertRevision } from "../../../src/next/store/engine.ts"; import { EXPORT_PAGE } from "../../../src/next/store/export.ts"; import { STORE_SCHEMA } from "../../../src/next/store/schema.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; @@ -32,7 +33,7 @@ afterEach(async () => { }); describe("parts", () => { - it("come in archive order: manifest, per table its table write, puts and indexes, the live documents, end", async () => { + it("come in archive order: manifest, per table its table write, puts and indexes, each record and its revisions, end", async () => { store = await archiveStore(); const parts = await exported(store); expect(parts[0]).toEqual({ @@ -40,11 +41,11 @@ describe("parts", () => { format: ARCHIVE_FORMAT, schema: STORE_SCHEMA, lastSeq: store.lastSeq, - nextDocumentId: store.nextDocumentId, + nextId: store.nextId, }); const kinds = parts.map((part) => part.kind === "put" || part.kind === "table" || part.kind === "index" - ? `${part.kind} ${part.table.split(":")[2]}` + ? `${part.kind} ${part.table.split(":")[1]}` : part.kind, ); const runs: [string, number][] = []; @@ -60,27 +61,32 @@ describe("parts", () => { ["index edits", 1], ["table notes", 1], ["put notes", 150], - ["document.create", 3], + ["document.create", 1], + ["document.revision", 1], + ["document.create", 1], + ["document.revision", 3], + ["document.create", 1], + ["document.revision", 2], ["end", 1], ]); expect(parts.at(-1)).toEqual({ kind: "end", writes: parts.length - 2 }); }); - it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, deltas folded into the base", async () => { + it("are lossless: stored values and versions, NULL cells absent, a migrating table's state, revisions as stored", async () => { store = await archiveStore(); const parts = await exported(store); const edits = parts.filter((part) => part.kind === "put" && part.table.endsWith(":edits")); expect(edits.slice(0, 2)).toEqual([ { kind: "put", - table: "client:app.store:edits", + table: "app.store:edits", row: { id: "e000", path: "p/0", at: 0, note: "n0", meta: { a: 0 }, done: false }, version: 1, columns: { meta: 1 }, }, { kind: "put", - table: "client:app.store:edits", + table: "app.store:edits", row: { id: "e001", path: "p/1", at: 1, done: false }, version: 1, columns: {}, @@ -92,20 +98,67 @@ describe("parts", () => { }); const notes = parts.filter((part) => part.kind === "put" && part.table.endsWith(":notes")); expect(notes.map((part) => (part.kind === "put" ? part.version : 0)).filter((v) => v === 2)).toHaveLength(100); - const documents = parts.filter((part) => part.kind === "document.create"); - expect( - documents.map((part) => - part.kind === "document.create" ? [part.record.key, part.record.revision, part.base.value] : [], - ), - ).toEqual([ + const documents = parts.flatMap((part): unknown[][] => + part.kind === "document.create" + ? [[part.record.key, part.record.revision, part.record.history]] + : part.kind === "document.revision" + ? [[part.content.kind, part.content.kind === "base" ? part.content.value : part.content.ops]] + : [], + ); + expect(documents).toEqual([ // A singleton's record has no key. - [undefined, 1, { title: "" }], - ["a", 3, { title: "a3" }], - ["b", 2, { title: "b2" }], + [undefined, 1, "none"], + ["base", { title: "" }], + ["a", 3, "none"], + ["base", { title: "a" }], + ["delta", [["s", ["title"], "a2"]]], + ["delta", [["s", ["title"], "a3"]]], + ["b", 2, "none"], + ["base", { title: "b" }], + ["delta", [["s", ["title"], "b2"]]], ]); }); }); +describe("revisions", () => { + it("a history longer than a page is read page by page and exported whole, in seq order", async () => { + store = await memoryStore(); + const count = EXPORT_PAGE * 2 + 1; + const record = { + id: 2, + owner: "app.store", + namespace: "session", + kind: "long", + history: "rewindable" as const, + createdAt: 1, + revision: 1, + version: 1, + fingerprint: "", + }; + const create: StagedWrite = { + order: "write", + apply: async (step) => { + await insertDocument(step, record, "{}"); + return []; + }, + }; + await store.commit([create], OPEN, context); + for (let n = 1; n < count; n++) { + const ops = JSON.stringify([["s", ["n"], n]]); + const delta: StagedWrite = { + order: "write", + apply: async (step) => { + await insertRevision(step, record.id, step.seq, "delta", 1, ops); + return []; + }, + }; + await store.commit([delta], OPEN, context); + } + const revisions = (await exported(store)).flatMap((part) => (part.kind === "document.revision" ? [part] : [])); + expect(revisions.map(({ seq }) => seq)).toEqual(Array.from({ length: count }, (_, index) => index + 1)); + }); +}); + describe("the commit queue", () => { it("is held from the first next() until the export ends: a commit waits", async () => { store = await archiveStore(); diff --git a/packages/chord/test/next/store/import.test.ts b/packages/chord/test/next/store/import.test.ts index d440a901e..02350a696 100644 --- a/packages/chord/test/next/store/import.test.ts +++ b/packages/chord/test/next/store/import.test.ts @@ -50,7 +50,7 @@ describe("round trips", () => { const store = await imported(parts); expect(await exported(store)).toEqual(parts); const manifest = parts[0] as Extract; - expect([store.lastSeq, store.nextDocumentId]).toEqual([manifest.lastSeq, manifest.nextDocumentId]); + expect([store.lastSeq, store.nextId]).toEqual([manifest.lastSeq, manifest.nextId]); }); it("the store keeps its ids, seqs and folded documents; the outbox, when asked, starts after lastSeq", async () => { @@ -59,8 +59,7 @@ describe("round trips", () => { expect(await store.outbox!.read(store.lastSeq, 10, context)).toEqual([]); expect(await rejection(store.outbox!.read(0, 10, context))).toMatchObject({ code: "invalid" }); const record = await store.read( - (executor) => - findDocument(executor, { namespace: "chord", owner: "client:app.store", kind: "memo", key: "a" }), + (executor) => findDocument(executor, { namespace: "chord", owner: "app.store", kind: "memo", key: "a" }), context, ); const original = parts.find((part) => part.kind === "document.create" && part.record.key === "a"); @@ -71,12 +70,12 @@ describe("round trips", () => { it("a migrating table resumes on the imported store; rows written at version 1 read as the table's", async () => { const store = await imported(await archive()); const { EditsV2, NotesV2 } = ArchiveTables; - const notes = "client:app.store:notes"; + const notes = "app.store:notes"; const fence = { admit: () => true, adopt: () => {} }; await expect( prepareTables([{ definition: NotesV2, table: notes }], store, fence, "app", context), ).rejects.toThrow("crash"); - const edits = new TableHandle(EditsV2, "client:app.store:edits", store, tableOwner()); + const edits = new TableHandle(EditsV2, "app.store:edits", store, tableOwner()); expect(await edits.get({ id: "e003" }, context)).toEqual({ id: "e003", path: "p/3", @@ -163,10 +162,20 @@ interface Mutable { } /** Part `index` as a writable object. */ +/** The positions of every part of `kind`. */ +function all(parts: readonly ArchivePart[], kind: ArchivePart["kind"]): number[] { + return parts.flatMap((part, index) => (part.kind === kind ? [index] : [])); +} + function at(parts: unknown[], index: number): Mutable { return parts[index] as Mutable; } +/** The content of the revision part at `index`, writable. */ +function revisionContent(parts: unknown[], index: number): Mutable { + return at(parts, index).content as Mutable; +} + describe("refusals", () => { it.each<[string, (parts: ArchivePart[]) => unknown[], string]>([ [ @@ -258,12 +267,12 @@ describe("refusals", () => { "tables come before the documents", ], [ - "a table name that is not kind:type:table", + "a table name that is not owner:table", (parts) => { at(parts, first(parts, "table")).table = "edits"; return parts; }, - "not kind:type:table", + "not owner:table", ], [ "a column name breaking the rules", @@ -325,8 +334,8 @@ describe("refusals", () => { [ "an id twice", (parts) => { - const document = first(parts, "document.create"); - (at(parts, document + 1).record as Mutable).id = (at(parts, document).record as Mutable).id!; + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).id = (at(parts, one!).record as Mutable).id!; return parts; }, "occurs twice", @@ -337,24 +346,75 @@ describe("refusals", () => { (at(parts, first(parts, "document.create")).record as Mutable).createdAt = 10_000; return parts; }, - "created after the manifest's lastSeq", + "a lifetime outside the manifest's lastSeq", ], [ - "a base at another version", + "a base at another version than its record's", (parts) => { - (at(parts, first(parts, "document.create")).base as Mutable).version = 2; + revisionContent(parts, first(parts, "document.revision")).version = 2; return parts; }, - "the base is not at the record's version", + "has no base at its version", + ], + [ + "a delta without a base at its version", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + revisionContent(parts, delta).version = 2; + return parts; + }, + "a delta without a base at its version", + ], + [ + "a revision out of seq order", + (parts) => { + const delta = parts.findIndex((part) => part.kind === "document.revision" && part.content.kind === "delta"); + at(parts, delta).seq = 1; + return parts; + }, + "out of seq order", + ], + [ + "a revision of another record", + (parts) => { + at(parts, first(parts, "document.revision")).id = 9_999; + return parts; + }, + "follows its record's create", + ], + [ + "a live record without revisions", + (parts) => parts.filter((_, index) => index !== first(parts, "document.revision")), + "has no base at its version", + ], + [ + "revisions of a retired none record", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.retiredAt = record.createdAt!; + return parts; + }, + "keeps no revisions", + ], + [ + "a retired rewindable record without its history", + (parts) => { + const record = at(parts, first(parts, "document.create")).record as Mutable; + record.history = "rewindable"; + record.retiredAt = record.createdAt!; + const id = record.id; + return parts.filter((part) => part.kind !== "document.revision" || part.id !== id); + }, + "has no base at its version", ], [ "two live records at one address", (parts) => { - const document = first(parts, "document.create"); - (at(parts, document + 1).record as Mutable).key = (at(parts, document).record as Mutable).key!; + const [one, two] = all(parts, "document.create"); + (at(parts, two!).record as Mutable).key = (at(parts, one!).record as Mutable).key!; return parts; }, - "a second live record at one address", + "a second record at one address for an overlapping lifetime", ], [ "a record's meta with a NUL", diff --git a/packages/chord/test/next/store/jsonl.test.ts b/packages/chord/test/next/store/jsonl.test.ts index 3d36e2321..9664737d3 100644 --- a/packages/chord/test/next/store/jsonl.test.ts +++ b/packages/chord/test/next/store/jsonl.test.ts @@ -17,8 +17,8 @@ async function collect(values: AsyncIterable): Promise { } const PARTS: ArchivePart[] = [ - { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextDocumentId: 2 }, - { kind: "put", table: "client:app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, + { kind: "manifest", format: 1, schema: 1, lastSeq: 3, nextId: 2 }, + { kind: "put", table: "app.store:edits", row: { id: "ü€😀", n: 1 }, version: 1, columns: {} }, { kind: "end", writes: 1 }, ]; diff --git a/packages/chord/test/next/store/outbox.test.ts b/packages/chord/test/next/store/outbox.test.ts index 2a720c080..4b5196524 100644 --- a/packages/chord/test/next/store/outbox.test.ts +++ b/packages/chord/test/next/store/outbox.test.ts @@ -14,13 +14,13 @@ import { type ChordStore, openStore } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { memoryStore, rejection } from "../support.ts"; -const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; const OPEN = { admit: () => true, adopt: () => {} }; /** Creates the singleton at `ADDRESS` (with `key`) in one commit. */ async function created(store: ChordStore, key = ""): Promise { - await store.commit([createDocument({ ...ADDRESS, key }, "latest", BASE, JSON.stringify(BASE.value))], OPEN, context); + await store.commit([createDocument({ ...ADDRESS, key }, BASE, JSON.stringify(BASE.value))], OPEN, context); } let directory: string | undefined; @@ -46,7 +46,7 @@ describe("the outbox", () => { await store.close(context); store = await open(true); expect(store.outbox?.floor).toBe(1); - const id = 1; + const id = 2; const ops = [["s", ["n"], 2]] as const; await store.commit( [changeDocument(id, 1, { kind: "delta", version: 1, ops }, JSON.stringify(ops))], @@ -58,6 +58,7 @@ describe("the outbox", () => { expect(await store.outbox!.read(1, 10, context)).toEqual([ { seq: 2, + nextId: 3, writes: [{ kind: "document.change", id, revision: 1, content: { kind: "delta", version: 1, ops } }], }, ]); @@ -75,7 +76,7 @@ describe("the outbox", () => { const [commit] = await store.outbox!.read(0, 10, context); expect(commit!.writes[0]).toMatchObject({ kind: "document.create", - record: { id: 1, createdAt: 1, ...ADDRESS }, + record: { id: 2, createdAt: 1, ...ADDRESS }, base: BASE, }); await store.outbox!.trim(1, context); diff --git a/packages/chord/test/next/store/query.test.ts b/packages/chord/test/next/store/query.test.ts index 9dfb93473..2c6875a5f 100644 --- a/packages/chord/test/next/store/query.test.ts +++ b/packages/chord/test/next/store/query.test.ts @@ -51,36 +51,32 @@ describe("cursors", () => { const order = query(T).orderBy((e) => e.at, "desc").order; it("round-trip the order values, also across conditions with the same order", () => { - const cursor = encodeCursor("client:app:edits", order, [5, "a\u{1F600}", 2]); + const cursor = encodeCursor("app:edits", order, [5, "a\u{1F600}", 2]); expect(cursor).toMatch(/^[A-Za-z0-9_-]+$/); - expect(decodeCursor(T, "client:app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); + expect(decodeCursor(T, "app:edits", order, cursor)).toEqual([5, "a\u{1F600}", 2]); const other = query(T) .where((e) => eq(e.id, "x")) .orderBy((e) => e.at, "desc").order; - expect(decodeCursor(T, "client:app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); + expect(decodeCursor(T, "app:edits", other, cursor)).toEqual([5, "a\u{1F600}", 2]); }); it("are a trust boundary: another table, order or direction, garbage and bad values are invalid/args", () => { - const cursor = encodeCursor("client:app:edits", order, [5, "a", 2]); - expect(() => decodeCursor(T, "client:app:edits", query(T).order, cursor)).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); + const cursor = encodeCursor("app:edits", order, [5, "a", 2]); + expect(() => decodeCursor(T, "app:edits", query(T).order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", query(T).orderBy((e) => e.at).order, cursor)).toThrow(args); expect(() => decodeCursor( defineTable("other", { version: 1, columns: { id: str(), entry: int(), at: int() }, key: ["id", "entry"] }), - "client:app:other", + "app:other", order, cursor, ), ).toThrow(args); // The same definition in another object's table. - expect(() => decodeCursor(T, "client:other:edits", order, cursor)).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", order, "%%%")).toThrow(args); - expect(() => decodeCursor(T, "client:app:edits", order, 42)).toThrow(args); - expect(() => - decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, ["x", "a", 2])), - ).toThrow(args); - expect(() => - decodeCursor(T, "client:app:edits", order, encodeCursor("client:app:edits", order, [5, "a"])), - ).toThrow(args); + expect(() => decodeCursor(T, "other:edits", order, cursor)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, "%%%")).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, 42)).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, ["x", "a", 2]))).toThrow(args); + expect(() => decodeCursor(T, "app:edits", order, encodeCursor("app:edits", order, [5, "a"]))).toThrow(args); }); }); diff --git a/packages/chord/test/next/store/refusal.test.ts b/packages/chord/test/next/store/refusal.test.ts new file mode 100644 index 000000000..760273954 --- /dev/null +++ b/packages/chord/test/next/store/refusal.test.ts @@ -0,0 +1,65 @@ +// Tests of refusals as Chord's owners take them (slice 6b D4, §1.4): an `int` CHECK fails only its transaction, any +// other refusal its object; the store's own errors pass through. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import { add } from "../../../src/next/store/expression.ts"; +import { refusalOf } from "../../../src/next/store/refusal.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { loaded, memoryStore, rejection } from "../support.ts"; + +describe("classification", () => { + it("a CHECK failure by SQLite's extended code or by its message; everything else broken; the store's own errors pass", () => { + const node = Object.assign(new Error("CHECK constraint failed: n"), { errcode: 275 }); + const other = Object.assign(new Error("UNIQUE constraint failed"), { errcode: 2067 }); + expect(refusalOf(node, "x")).toMatchObject({ kind: "check", error: { code: "invalid", reason: "args" } }); + expect(refusalOf(new Error("CHECK constraint failed: n"), "x")).toMatchObject({ kind: "check" }); + expect(refusalOf(other, "x")).toMatchObject({ kind: "broken", error: { code: "unavailable" } }); + const stopped = new ChordError({ + code: "unavailable", + reason: "storage_failed", + message: "store: storage failed", + }); + expect(refusalOf(stopped, "x")).toBeUndefined(); + expect(refusalOf(new ChordError({ code: "released", message: "closed" }), "x")).toBeUndefined(); + }); +}); + +describe("Chord's owners", () => { + const Counts = defineTable("counts", { version: 1, columns: { id: str(), n: int() }, key: ["id"] }); + const Counter = local("app.counter", { + version: 1, + params: {}, + members: { bump: method(obj({ by: int() })), reset: method() }, + }); + + it("an int out of range fails only its transaction; the object stays live", async () => { + const store = await memoryStore(); + const { host, errors } = await loaded( + (h) => + h.serve(Counter, (_, self) => { + const counts = self.table(Counts); + return { + bump: async ({ by }, ctx) => { + await self.tx(async (tx) => tx.update(counts, { id: "a" }, (c) => ({ n: add(c.n, by) })), ctx); + return null; + }, + reset: async (ctx) => { + await self.tx(async (tx) => tx.put(counts, { id: "a", n: Number.MAX_SAFE_INTEGER }), ctx); + return null; + }, + }; + }), + { store }, + ); + const counter = await host.get(Counter, {}, context); + await counter.reset(context); + expect(await rejection(counter.bump({ by: 1 }, context))).toMatchObject({ code: "invalid", reason: "args" }); + expect(counter.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + await counter.bump({ by: -1 }, context); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/store/sql.test.ts b/packages/chord/test/next/store/sql.test.ts index e8adaa0eb..38b0c8623 100644 --- a/packages/chord/test/next/store/sql.test.ts +++ b/packages/chord/test/next/store/sql.test.ts @@ -29,7 +29,7 @@ const T = defineTable("edits", { describe("names", () => { it("renders ::

, quoted", () => { - expect(tableSqlName("session", "@pi/file-history.store", "edits")).toBe("session:@pi/file-history.store:edits"); + expect(tableSqlName("@pi/file-history.store", "edits")).toBe("@pi/file-history.store:edits"); expect(quote("session:a:b")).toBe('"session:a:b"'); }); }); diff --git a/packages/chord/test/next/store/store.test.ts b/packages/chord/test/next/store/store.test.ts index 03f1c0087..e94641317 100644 --- a/packages/chord/test/next/store/store.test.ts +++ b/packages/chord/test/next/store/store.test.ts @@ -1,19 +1,22 @@ -// The store (slice 6a §2, §3): opening, the commit step, storage failures, one host per store (rule 7), close. +// The store (slice 6a §2, §3, slice 6b §1): opening, the commit step, refusals and unknown outcomes (D4), the id +// allocator (D3), one host per store (rule 7), close. import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { isChordError } from "../../../src/next/data/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; import { sqliteBackend } from "../../../src/next/store/backend.ts"; import { changeDocument, createDocument, retireDocument } from "../../../src/next/store/engine.ts"; import { importStore } from "../../../src/next/store/import.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; import { type ChordStore, openStore, type Store } from "../../../src/next/store/store.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; -import { exported, metered, rejection } from "../support.ts"; +import { exported, faultyStore, holdNextCommit, metered, rejection } from "../support.ts"; -const ADDRESS = { namespace: "chord", owner: "session:app.store", kind: "notes", key: "" }; +const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; /** A store on a fresh `:memory:` database, as Chord's layers see it. */ @@ -32,15 +35,15 @@ function recording(adopted: [number, readonly number[]][] = []) { /** Creates the singleton at `ADDRESS` and resolves with its id. */ async function created(store: ChordStore): Promise { const { adopted, fence } = recording(); - await store.commit([createDocument(ADDRESS, "latest", BASE, JSON.stringify(BASE.value))], fence, context); + await store.commit([createDocument(ADDRESS, BASE, JSON.stringify(BASE.value))], fence, context); return adopted[0]![1][0]!; } -/** A staged write whose step fails in the backend. */ +/** A staged write whose step fails in the backend: its own code throws, a refusal. */ const BROKEN: StagedWrite = { order: "write", apply: (step) => - step.run("INSERT INTO no_such_table VALUES (1)").then(() => ({ kind: "document.retire", id: 0, revision: 0 })), + step.run("INSERT INTO no_such_table VALUES (1)").then(() => [{ kind: "document.retire", id: 0, revision: 0 }]), }; let directory: string | undefined; @@ -56,13 +59,13 @@ describe("opening", () => { const path = join(directory, "store.sqlite"); const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; expect([store.lastSeq, store.outbox]).toEqual([0, undefined]); - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); await store.close(context); const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; expect(reopened.lastSeq).toBe(1); const { adopted, fence } = recording(); - await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], fence, context); - expect(adopted).toEqual([[2, [2]]]); + await reopened.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [3]]]); await reopened.close(context); }); @@ -84,24 +87,24 @@ describe("the commit step", () => { adopt: (seq: number, ids: readonly number[]) => void events.push(`${name} ${seq} ${ids}`), }); const first = store.commit( - [createDocument(ADDRESS, "latest", BASE, "{}"), retireDocument(id, 1)], + [createDocument(ADDRESS, BASE, "{}"), retireDocument(id, 1)], fence("recreate"), context, ); const second = store.commit( - [changeDocument(2, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], + [changeDocument(3, 1, { kind: "delta", version: 1, ops: [["s", ["n"], 1]] }, '[["s",["n"],1]]')], fence("change"), context, ); expect(await Promise.all([first, second])).toEqual([2, 3]); - expect(events).toEqual(["recreate 2 2", "change 3 "]); + expect(events).toEqual(["recreate 2 3", "change 3 "]); expect(store.lastSeq).toBe(3); }); it("rejects a commit its fence refuses at the entry released, writing nothing; an empty commit takes no seq", async () => { const store = await memoryStore(); const refused = store.commit( - [createDocument(ADDRESS, "latest", BASE, "{}")], + [createDocument(ADDRESS, BASE, "{}")], { admit: () => false, adopt: () => {} }, context, ); @@ -109,7 +112,7 @@ describe("the commit step", () => { const { adopted, fence } = recording(); expect(await store.commit([], fence, context)).toBe(0); expect(adopted).toEqual([[0, []]]); - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); }); it("numbers up to Number.MAX_SAFE_INTEGER after an import near it, then fails storage_failed: no seq repeats", async () => { @@ -120,13 +123,9 @@ describe("the commit step", () => { parts, context, )) as ChordStore; - expect(await created(store)).toBe(1); + expect(await created(store)).toBe(2); expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); - const refused = store.commit( - [createDocument({ ...ADDRESS, key: "b" }, "latest", BASE, "{}")], - recording().fence, - context, - ); + const refused = store.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], recording().fence, context); expect(await rejection(refused)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); expect(store.lastSeq).toBe(Number.MAX_SAFE_INTEGER); expect(await rejection(store.read(async () => null, context))).toMatchObject({ reason: "storage_failed" }); @@ -151,26 +150,49 @@ describe("the commit step", () => { }); }); -describe("storage failures", () => { - it("stop the store: the commit rejects storage_failed, later commits and reads too; the host is told once", async () => { +describe("refusals and unknown outcomes (D4)", () => { + it("a step whose own code throws is refused: the commit rejects with that error, nothing is adopted, the store goes on", async () => { const store = await memoryStore(); const told: unknown[] = []; - store.attach((error) => told.push(error)); - expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ - code: "unavailable", - reason: "storage_failed", - }); - expect(await rejection(store.commit([BROKEN], recording().fence, context))).toMatchObject({ - reason: "storage_failed", - }); - expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); - expect(told.length).toBe(1); - expect(store.lastSeq).toBe(0); - await store.close(context); - await store.close(context); + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + const { adopted, fence } = recording(); + const error = await rejection(store.commit([BROKEN], fence, context)); + expect(String(error)).toContain("no such table"); + expect(isChordError(error)).toBe(false); + expect([adopted, told, store.lastSeq]).toEqual([[], [], 0]); + expect(await created(store)).toBe(2); + expect(store.lastSeq).toBe(1); }); - it("include an adopt that throws and a conditional update that finds another revision", async () => { + it.each([ + ["BEGIN IMMEDIATE", undefined], + ["COMMIT", undefined], + ["ROLLBACK", BROKEN], + ] as const)( + "a failed %s is an unknown outcome: the store stops storage_failed, its host told once", + async (at, write) => { + const { store, fail } = await faultyStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + fail(at); + const writes = write === undefined ? [createDocument(ADDRESS, BASE, "{}")] : [write]; + expect(await rejection(store.commit(writes, recording().fence, context))).toMatchObject({ + code: "unavailable", + reason: "storage_failed", + }); + expect( + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)), + ).toMatchObject({ + reason: "storage_failed", + }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + expect(told.length).toBe(1); + await store.close(context); + await store.close(context); + }, + ); + + it("an adopt that throws stops the store, an empty commit's too; a conditional update on another revision is refused", async () => { const store = await memoryStore(); const id = await created(store); const throwing = { @@ -182,40 +204,109 @@ describe("storage failures", () => { expect(await rejection(store.commit([retireDocument(id, 1)], throwing, context))).toMatchObject({ reason: "storage_failed", }); - // An empty commit's adopt too. const empty = await memoryStore(); expect(await rejection(empty.commit([], throwing, context))).toMatchObject({ reason: "storage_failed" }); expect(await rejection(empty.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); const other = await memoryStore(); const otherId = await created(other); const stale = changeDocument(otherId, 7, { kind: "delta", version: 1, ops: [] }, "[]"); - expect(await rejection(other.commit([stale], recording().fence, context))).toMatchObject({ - reason: "storage_failed", - }); + expect(String(await rejection(other.commit([stale], recording().fence, context)))).toContain( + "not live at revision 7", + ); expect(other.lastSeq).toBe(1); + expect(await other.read(async () => 1, context)).toBe(1); }); it("spare a read whose context was aborted (aborted), and reject a read that settles after the stop", async () => { - const store = await memoryStore(); + const { store, fail } = await faultyStore(); const cancel = withCancel(context); cancel.cancel(); const failing = store.read(() => Promise.reject(new Error("backend")), cancel.context); expect(await rejection(failing)).toMatchObject({ code: "aborted" }); const gate = Promise.withResolvers(); const late = store.read(() => gate.promise, context); - await rejection(store.commit([BROKEN], recording().fence, context)); + fail("COMMIT"); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context)); gate.resolve(1); expect(await rejection(late)).toMatchObject({ reason: "storage_failed" }); }); + + it("runs a read transaction's queries on one state, guarded as a read", async () => { + const store = await memoryStore(); + await created(store); + const counts = await store.readTransaction( + async (executor) => [ + await executor.get("SELECT count(*) AS n FROM chord_documents"), + await executor.get("SELECT 1 AS one"), + ], + context, + ); + expect(counts).toEqual([{ n: 1 }, { one: 1 }]); + expect(await rejection(store.readTransaction(() => Promise.reject(new Error("backend")), context))).toMatchObject( + { + reason: "storage_failed", + }, + ); + }); +}); + +describe("the id allocator (D3)", () => { + /** A staged write raising the allocator above `id`, as an adapter's explicit id does. */ + const raising = (id: number): StagedWrite => ({ + order: "write", + apply: async (step) => { + step.ids.raise(id); + return []; + }, + }); + + it("starts at 2, takes a creation's id as it applies, and gives a mint during a running step the next one", async () => { + const store = await memoryStore({ outbox: true }); + const held = holdNextCommit(store); + const commit = store.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + await held.entered; + expect(store.mintId()).toBe(3); + held.release(); + await commit; + expect(store.mintId()).toBe(4); + const [first] = await store.outbox!.read(0, 10, context); + expect([first!.nextId, (first!.writes[0] as { record: { id: number } }).record.id]).toEqual([4, 2]); + expect(store.nextId).toBe(4); + }); + + it("leaves a refused step's ids unused and keeps the raised value", async () => { + const store = await memoryStore(); + await rejection(store.commit([createDocument(ADDRESS, BASE, "{}"), BROKEN], recording().fence, context)); + expect(await created(store)).toBe(3); + }); + + it("saturates at Number.MAX_SAFE_INTEGER across reopen: mints and Chord creations are refused, the store goes on", async () => { + directory = await mkdtemp(join(tmpdir(), "chord-store-")); + const path = join(directory, "store.sqlite"); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + await store.commit([raising(Number.MAX_SAFE_INTEGER)], recording().fence, context); + expect(store.nextId).toBe(Number.MAX_SAFE_INTEGER); + await store.close(context); + const reopened = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + expect(reopened.nextId).toBe(Number.MAX_SAFE_INTEGER); + expect(() => reopened.mintId()).toThrow("ID space is exhausted"); + const refused = reopened.commit([createDocument(ADDRESS, BASE, "{}")], recording().fence, context); + expect(String(await rejection(refused))).toContain("ID space is exhausted"); + expect(await reopened.read(async () => 1, context)).toBe(1); + await reopened.close(context); + }); }); describe("one host per store (rule 7)", () => { - it("refuses a second listener until the first detaches", async () => { + it("refuses a second attachment until the first detaches", async () => { const store = await memoryStore(); - store.attach(() => {}); - expect(() => store.attach(() => {})).toThrow("already attached"); + const attachment = { failed: () => {}, batch: new Batch(() => {}, undefined) }; + store.attach(attachment); + expect(store.batch).toBe(attachment.batch); + expect(() => store.attach(attachment)).toThrow("already attached"); store.detach(); - store.attach(() => {}); + expect(store.batch).toBeUndefined(); + store.attach(attachment); }); }); @@ -224,7 +315,7 @@ describe("close", () => { const store = await memoryStore(); const gate = Promise.withResolvers(); const entered = Promise.withResolvers(); - const create = createDocument(ADDRESS, "latest", BASE, "{}"); + const create = createDocument(ADDRESS, BASE, "{}"); const held: StagedWrite = { order: "create", apply: async (step) => { @@ -235,11 +326,7 @@ describe("close", () => { }; const running = store.commit([held], recording().fence, context); await entered.promise; - const queued = store.commit( - [createDocument({ ...ADDRESS, key: "q" }, "latest", BASE, "{}")], - recording().fence, - context, - ); + const queued = store.commit([createDocument({ ...ADDRESS, key: "q" }, BASE, "{}")], recording().fence, context); const closed = (store as Store).close(context); gate.resolve(); expect(await running).toBe(1); diff --git a/packages/chord/test/next/store/table-changes.test.ts b/packages/chord/test/next/store/table-changes.test.ts index 2f51e3e37..8f3e75047 100644 --- a/packages/chord/test/next/store/table-changes.test.ts +++ b/packages/chord/test/next/store/table-changes.test.ts @@ -43,7 +43,7 @@ describe("staging", () => { changes.delete({ id: "e9" }); expect(staged).toHaveLength(3); expect( - await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "session:app.store:edits"`), context), + await store.read((executor) => executor.get(`SELECT count(*) AS n FROM "app.store:edits"`), context), ).toEqual({ n: 1 }); }); }); diff --git a/packages/chord/test/next/store/table-handle.test.ts b/packages/chord/test/next/store/table-handle.test.ts index 7a51ef960..0a572d37c 100644 --- a/packages/chord/test/next/store/table-handle.test.ts +++ b/packages/chord/test/next/store/table-handle.test.ts @@ -125,12 +125,12 @@ describe("unreadable cells", () => { it("a json cell of a newer version fails the owner and the read, the data intact", async () => { const owner = tableOwner(); const edits = await filled(1, owner); - await store.read((executor) => executor.run(`UPDATE "session:app.store:edits" SET "meta$v" = 2`), context); + await store.read((executor) => executor.run(`UPDATE "app.store:edits" SET "meta$v" = 2`), context); const error = await rejection(edits.get({ id: "e000" }, context)); expect(error).toMatchObject({ code: "unavailable" }); expect(owner.failures).toHaveLength(1); - expect( - await store.read((executor) => executor.get(`SELECT meta FROM "session:app.store:edits"`), context), - ).toEqual({ meta: '{"a":0}' }); + expect(await store.read((executor) => executor.get(`SELECT meta FROM "app.store:edits"`), context)).toEqual({ + meta: '{"a":0}', + }); }); }); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts index c55ae4b96..d58bef1b7 100644 --- a/packages/chord/test/next/store/table-migration.test.ts +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -17,7 +17,7 @@ import { TableHandle } from "../../../src/next/store/table-handle.ts"; import { MIGRATION_CHUNK, prepareTables } from "../../../src/next/store/table-migration.ts"; import { memoryStore, rejection, tableOwner } from "../support.ts"; -const TABLE = "client:app.store:edits"; +const TABLE = "app.store:edits"; const OPEN = { admit: () => true, adopt: () => {} }; const EditsV1 = defineTable("edits", { diff --git a/packages/chord/test/next/store/table-writes.test.ts b/packages/chord/test/next/store/table-writes.test.ts index d1bd06c0a..7fae21403 100644 --- a/packages/chord/test/next/store/table-writes.test.ts +++ b/packages/chord/test/next/store/table-writes.test.ts @@ -26,7 +26,7 @@ const Edits = defineTable("edits", { key: ["id"], indexes: { by_path: ["path"] }, }); -const TABLE = "session:app.store:edits"; +const TABLE = "app.store:edits"; const FENCE = { admit: () => true, adopt: () => {} }; let store: ChordStore; @@ -71,10 +71,11 @@ describe("schema writes", () => { expect(indexes).toContainEqual({ name: `${TABLE}:by_path` }); }); - it("an index write for a table the store does not have fails the step (storage_failed)", async () => { + it("an index write for a table the store does not have refuses the step (D4: the store goes on)", async () => { await opened(); - const write = stageIndex({ kind: "index", table: "session:app.store:nope", name: "x", columns: ["id"] }); - await expect(store.commit([write], FENCE, context)).rejects.toMatchObject({ reason: "storage_failed" }); + const write = stageIndex({ kind: "index", table: "app.store:nope", name: "x", columns: ["id"] }); + await expect(store.commit([write], FENCE, context)).rejects.toThrow("not in the store"); + expect(await store.read(async () => 1, context)).toBe(1); }); }); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 7ad2b9cd7..428dea36f 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -1,6 +1,7 @@ // Test support: cost counters through the in-memory telemetry adapter, GC checks, and hosts, links, watches and // rejections for the slice tests. +import { DatabaseSync } from "node:sqlite"; import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; @@ -65,7 +66,7 @@ import { evolveTable } from "../../src/next/store/table-evolve.ts"; import { TableHandle, type TableOwner } from "../../src/next/store/table-handle.ts"; import { prepareTables } from "../../src/next/store/table-migration.ts"; import { nodeCompression } from "../../src/next-node/compression.ts"; -import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; import type { NativeWatchHandle, WatchCall, WatchHandle } from "../../src/watch.ts"; /** Counts of the `chord.data.*` counters. */ @@ -158,7 +159,7 @@ export function valuesOf(calls: readonly WatchCall[]): (T | { readonly fai */ export function failLive(objects: readonly WeakRef[], self: Self, error: ChordError): void { // The factory's `self` is its attempt. - for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, error); + for (const object of objects) object.deref()?.failLive(self as unknown as Attempt, { kind: "broken", error }); } /** A local type whose objects' attempts a test ends by hand (`restartable`); it ends with its last reference. */ @@ -252,6 +253,37 @@ export async function memoryStore(options?: { readonly outbox?: boolean }): Prom return (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore; } +/** A statement of the Node facade's transactions a test can make fail. */ +export type TransactionStatement = "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK"; + +/** + * A Node SQLite facade on `:memory:` whose next `statement` throws, once per `fail` (the D4 matrix: a failed + * `BEGIN`, `COMMIT` or `ROLLBACK`, as the real facade sees one). + */ +export function faultyDatabase(): { readonly db: NodeSqliteDatabase; fail(statement: TransactionStatement): void } { + const database = new DatabaseSync(":memory:"); + const failing = new Set(); + const exec = database.exec.bind(database); + database.exec = (sql: string): void => { + if (!failing.delete(sql)) { + exec(sql); + return; + } + // A failed `ROLLBACK` here still rolls back, so the database stays usable for the rest of the test. + if (sql === "ROLLBACK") exec(sql); + throw new Error(`injected: ${sql} failed`); + }; + return { db: new NodeSqliteDatabase(database), fail: (statement) => void failing.add(statement) }; +} + +/** A store over `faultyDatabase()`, and its `fail`. */ +export async function faultyStore(options?: { + readonly outbox?: boolean; +}): Promise<{ readonly store: ChordStore; fail(statement: TransactionStatement): void }> { + const { db, fail } = faultyDatabase(); + return { store: (await openStore(sqliteBackend(db), BACKGROUND_CONTEXT, options)) as ChordStore, fail }; +} + /** * Holds `store`'s next commit step inside its backend transaction, after its admission: `entered` resolves there, * `release()` lets it finish; later commits run as usual. @@ -267,7 +299,7 @@ export function holdNextCommit(store: ChordStore): { readonly entered: Promise { entered.resolve(); await gate.promise; - return { kind: "dropIndex", table: "x", name: "y" }; + return [{ kind: "dropIndex", table: "x", name: "y" }]; }, }; return commit([...writes, held], fence, ctx); @@ -280,13 +312,13 @@ export function tableOwner(failures: ChordError[] = []): TableOwner & { readonly return { label: "app.store", scope: {}, assertLive: () => {}, fail: (error) => void failures.push(error), failures }; } -/** Creates `definition` (and its indexes) in `store` as `session:app.store:` and returns a handle to it. */ +/** Creates `definition` (and its indexes) in `store` as `app.store:` and returns a handle to it. */ export async function tableIn( store: ChordStore, definition: TableDefinition, owner = tableOwner(), ): Promise { - const table = `session:app.store:${definition.name}`; + const table = `app.store:${definition.name}`; const fence = { admit: () => true, adopt: () => {} }; await prepareTables([{ definition, table }], store, fence, "app.store", BACKGROUND_CONTEXT); return new TableHandle(definition, table, store, owner); @@ -886,8 +918,8 @@ export const ArchiveTables = (() => { })(); /** - * A store holding a bit of everything an archive carries: `client:app.store:edits` at version 2 with 150 rows (two - * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `client:app.store:notes` left + * A store holding a bit of everything an archive carries: `app.store:edits` at version 2 with 150 rows (two + * export pages; some written at version 1, `NULL` and `json(...)` cells, an index), `app.store:notes` left * `migrating` at version 2 after one committed chunk, and documents: a singleton, keyed ones with deltas after their * base, and a retired one. */ @@ -895,8 +927,8 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro const { EditsV1, EditsV2, NotesV1, NotesV2 } = ArchiveTables; const store = await memoryStore(options); const fence = { admit: () => true, adopt: () => {} }; - const edits = await tableAt(store, "client:app.store:edits", EditsV1); - const notes = await tableAt(store, "client:app.store:notes", NotesV1); + const edits = await tableAt(store, "app.store:edits", EditsV1); + const notes = await tableAt(store, "app.store:notes", NotesV1); const staged: StagedWrite[] = []; const editChanges = new TableChanges(edits, staged); for (let i = 0; i < 150; i++) { @@ -911,21 +943,15 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro const noteChanges = new TableChanges(notes, staged); for (let i = 0; i < 150; i++) noteChanges.put({ id: i, text: `t${i}` }); await store.commit(staged, fence, BACKGROUND_CONTEXT); + await prepareTables([{ definition: EditsV2, table: "app.store:edits" }], store, fence, "app", BACKGROUND_CONTEXT); await prepareTables( - [{ definition: EditsV2, table: "client:app.store:edits" }], - store, - fence, - "app", - BACKGROUND_CONTEXT, - ); - await prepareTables( - [{ definition: NotesV2, table: "client:app.store:notes" }], + [{ definition: NotesV2, table: "app.store:notes" }], store, fence, "app", BACKGROUND_CONTEXT, ).catch(() => {}); - const address = { namespace: "chord", owner: "client:app.store", kind: "memo" }; + const address = { namespace: "chord", owner: "app.store", kind: "memo" }; const base = (value: JsonValue) => ({ version: 1, fingerprint: "f1", value }); let ids: readonly number[] = []; const created = { @@ -936,7 +962,7 @@ export async function archiveStore(options?: { readonly outbox?: boolean }): Pro }; await store.commit( ["", "a", "b", "gone"].map((key) => - createDocument({ ...address, key }, "latest", base({ title: key }), JSON.stringify({ title: key })), + createDocument({ ...address, key }, base({ title: key }), JSON.stringify({ title: key })), ), created, BACKGROUND_CONTEXT, diff --git a/packages/chord/test/sqlite/cloudflare.test.ts b/packages/chord/test/sqlite/cloudflare.test.ts index 41a270524..1202d8f49 100644 --- a/packages/chord/test/sqlite/cloudflare.test.ts +++ b/packages/chord/test/sqlite/cloudflare.test.ts @@ -1,5 +1,10 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT } from "../../src/context/index.ts"; +import { sqliteBackend } from "../../src/next/store/backend.ts"; +import { refusalOf } from "../../src/next/store/refusal.ts"; +import type { StagedWrite } from "../../src/next/store/step.ts"; +import { type ChordStore, openStore } from "../../src/next/store/store.ts"; import { DurableObjectSqliteDatabase } from "../../src/sqlite/cloudflare.ts"; import { durableObjectStorage, RUN_CHANGES, runChanges } from "./helpers.ts"; @@ -123,3 +128,72 @@ describe("Durable Object SQLite driver", () => { expect(sqlite.prepare("SELECT id FROM items").all()).toEqual([]); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** The test storage, its `transaction` failing after the closure resolved (a failed commit) when asked. */ + function failingCommit(): { storage: ReturnType; fail(): void } { + const storage = durableObjectStorage(); + const transaction = storage.transaction.bind(storage); + let failing = false; + return { + storage: { + ...storage, + transaction: async (closure) => { + const result = await transaction(closure); + if (!failing) return result; + failing = false; + throw new Error("commit failed"); + }, + }, + fail: () => { + failing = true; + }, + }; + } + + it("rejects with the callback's own error after a rollback, another one when the commit fails", async () => { + const { storage, fail } = failingCommit(); + const database = new DurableObjectSqliteDatabase(storage); + const own = new Error("refused"); + await expect(database.transaction(async () => Promise.reject(own))).rejects.toBe(own); + fail(); + await expect(database.transaction(async () => 1)).rejects.toThrow("commit failed"); + }); + + it("a store on it tells a CHECK refusal by its message, its error having no extended code", async () => { + const storage = durableObjectStorage(); + const exec = storage.sql.exec.bind(storage.sql); + // The Durable Object binding's errors carry a message only. + const withoutCodes: typeof storage = { + ...storage, + sql: { + exec: (query, ...bindings) => { + try { + return exec(query, ...bindings); + } catch (error) { + throw new Error((error as Error).message); + } + }, + }, + }; + const store = (await openStore( + sqliteBackend(new DurableObjectSqliteDatabase(withoutCodes)), + BACKGROUND_CONTEXT, + )) as ChordStore; + await store.read( + (executor) => executor.exec('CREATE TABLE "checked" (n INTEGER CHECK (n < 10))'), + BACKGROUND_CONTEXT, + ); + const refused: StagedWrite = { + order: "write", + apply: async (step) => { + await step.run('INSERT INTO "checked" VALUES (11)'); + return []; + }, + }; + const fence = { admit: () => true, adopt: () => {} }; + const error = await store.commit([refused], fence, BACKGROUND_CONTEXT).catch((thrown: unknown) => thrown); + expect(refusalOf(error, "x")).toMatchObject({ kind: "check" }); + expect(await store.read(async () => 1, BACKGROUND_CONTEXT)).toBe(1); + }); +}); diff --git a/packages/chord/test/sqlite/node.test.ts b/packages/chord/test/sqlite/node.test.ts index 96ad09a96..8666b3f5d 100644 --- a/packages/chord/test/sqlite/node.test.ts +++ b/packages/chord/test/sqlite/node.test.ts @@ -1,7 +1,7 @@ import { DatabaseSync } from "node:sqlite"; import { describe, expect, it, vi } from "vitest"; import type { SqliteExecutor } from "../../src/sqlite/index.ts"; -import { openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; +import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../../src/sqlite/node.ts"; import { RUN_CHANGES, runChanges } from "./helpers.ts"; describe("Node SQLite driver", () => { @@ -198,3 +198,48 @@ describe("Node SQLite driver", () => { await database.close(); }); }); + +describe("the outcome of a transaction (slice 6b D4)", () => { + /** A facade on `:memory:` whose next `statement` throws (and a `ROLLBACK` still rolls back). */ + function faulty(statement?: "BEGIN IMMEDIATE" | "COMMIT" | "ROLLBACK") { + const database = new DatabaseSync(":memory:"); + const exec = database.exec.bind(database); + let failing = statement; + database.exec = (sql: string): void => { + if (sql !== failing) { + exec(sql); + return; + } + failing = undefined; + if (sql === "ROLLBACK") exec(sql); + throw new Error(`${sql} failed`); + }; + return new NodeSqliteDatabase(database); + } + + it("rejects with the callback's own error after a rollback: the caller knows it never committed", async () => { + const database = faulty(); + const own = new Error("refused"); + await database.exec("CREATE TABLE t (n INTEGER)"); + const outcome = database.transaction(async (handle) => { + await handle.run("INSERT INTO t VALUES (1)"); + throw own; + }); + await expect(outcome).rejects.toBe(own); + expect(await database.all("SELECT n FROM t")).toEqual([]); + }); + + it.each(["BEGIN IMMEDIATE", "COMMIT"] as const)("rejects with another error when %s fails", async (statement) => { + const database = faulty(statement); + const outcome = database.transaction(async () => 1); + await expect(outcome).rejects.toThrow(`${statement} failed`); + }); + + it("rejects with an AggregateError, not the callback's error, when its ROLLBACK fails", async () => { + const database = faulty("ROLLBACK"); + const own = new Error("refused"); + const error = await database.transaction(async () => Promise.reject(own)).catch((thrown: unknown) => thrown); + expect(error).toBeInstanceOf(AggregateError); + expect(error).not.toBe(own); + }); +}); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 46658afba..56763148d 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -100,6 +100,7 @@ ### Added +- `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and commits each batch as one write of the store's commit step, adopted in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. - Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). diff --git a/packages/durable/README.md b/packages/durable/README.md index bdf59282e..d87c54d34 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -956,6 +956,8 @@ node --conditions=source --experimental-strip-types test/examples/13-chat.ts | [32-tests-after-edits](test/examples/32-tests-after-edits.ts) | An `afterRun` hook that runs the tests after every run that edited files, a fix run when they fail, and input waiting for them | | [33-turn-limit-and-continue](test/examples/33-turn-limit-and-continue.ts) | A per-conversation turn limit, `runFailure`, and continuing without a user message | | [34-submit-in-commit](test/examples/34-submit-in-commit.ts) | Admitting a visitor's steer inside the host's own commit with `submit(tx, draft)` and `whenIdle: "reject"` | +| [35-durable-on-the-store](test/examples/35-durable-on-the-store.ts) | A Harness on `StoreStorage` in a Chord host's store, next to a Chord object's own table: one commit queue, one id allocator, one export | +| [36-durable-views](test/examples/36-durable-views.ts) | A Chord object following a Durable view with `self.follow`: its frames and a member reading the followed value | | [00](test/examples/00-conversation.ts)–[12](test/examples/12-recovery.ts) | The layers underneath: sessions, documents, forks, watches, the Harness, agent configuration, reload, extension state, tasks, recovery | Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider otherwise. diff --git a/packages/durable/package.json b/packages/durable/package.json index 106512d21..933fe6db7 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -84,6 +84,12 @@ "import": "./dist/storage/jsonl/node.js", "default": "./dist/storage/jsonl/node.js" }, + "./storage/store": { + "source": "./src/storage/store/index.ts", + "types": "./dist/storage/store/index.d.ts", + "import": "./dist/storage/store/index.js", + "default": "./dist/storage/store/index.js" + }, "./storage/sqlite": { "source": "./src/storage/sqlite/index.ts", "types": "./dist/storage/sqlite/index.d.ts", @@ -119,7 +125,8 @@ "scripts": { "clean": "shx rm -rf dist", "build": "tsc -p tsconfig.build.json", - "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:workers", + "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:store && npm run test:workers", + "test:store": "vitest --run --config vitest.store.config.ts", "test:workers": "vitest --run --config test-workers/vitest.config.ts", "bench:storage": "vitest bench --config vitest.benchmark.config.ts", "bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts", diff --git a/packages/durable/src/storage/store/commit.ts b/packages/durable/src/storage/store/commit.ts new file mode 100644 index 000000000..cc53f5418 --- /dev/null +++ b/packages/durable/src/storage/store/commit.ts @@ -0,0 +1,327 @@ +// One Durable commit as one composite write of the store's commit step (slice 6b §2.3): staged before the queue (rows +// encoded, contents serialized, explicit ids raising the store's allocator, fork copies resolved from their sources), +// then inside the step what `SqliteStorage` does inside its transaction, with the step's reader: Durable's checks of +// ids and documents (its messages), the table puts, and the documents in two passes, existing ones first (their content, +// then their retirement, which frees the addresses this commit's creations take), then the creations and copies. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type EncodedRow, + type DocumentContent as EngineContent, + type DocumentRecord as EngineRecord, + findDocument, + insertDocument, + type OwnerTable, + readDocument, + retireRecord, + type StagedWrite, + type Step, + type StoreWrite, + writeContent, +} from "@earendil-works/chord/next"; +import type { + DocumentContent, + DocumentCopySource, + DocumentCreate, + DocumentId, + DocumentRecord, + Id, + JsonObject, + StorageWrite, +} from "../../types.ts"; +import { addressOf, durableRecord, engineRecord } from "./documents.ts"; +import { conversationRow, entryRow, submissionRow, taskRow } from "./tables.ts"; + +/** Durable's record types, as `record_ids` names them. */ +export type RecordType = "conversation" | "entry" | "task" | "submission" | "document"; + +/** The storage's tables, as a commit writes them. */ +export type CommitTables = { + readonly recordIds: OwnerTable; + readonly conversations: OwnerTable; + readonly entries: OwnerTable; + readonly entryHeads: OwnerTable; + readonly tasks: OwnerTable; + readonly submissions: OwnerTable; +}; + +/** A table write of the commit: its record's id and type, its table and its row encoded at staging. */ +type RowWrite = { + readonly id: number; + readonly type: Exclude; + readonly table: OwnerTable; + readonly row: EncodedRow; +}; + +/** A base as the commit stores it: its version and value, and the value serialized. */ +type StagedBase = { readonly version: number; readonly value: JsonObject; readonly text: string }; + +/** What one commit does with one document, as `SqliteStorage`'s action map has it. */ +type DocumentAction = { + create?: DocumentCreate; + copy?: DocumentCopySource; + /** A creation's base: the write's own, or a copy's once resolved. */ + base?: StagedBase; + content?: { readonly content: DocumentContent; readonly text: string }; + retire: boolean; +}; + +/** Reads a copy's source at its point: its stored document, `undefined` when it cannot be read there. */ +export type CopySource = ( + source: DocumentCopySource, +) => Promise<{ readonly record: DocumentRecord; readonly version: number; readonly value: JsonObject } | undefined>; + +/** A Durable commit staged for the store's step. */ +export class DurableCommit implements StagedWrite { + readonly order = "write"; + readonly #owner: string; + readonly #tables: CommitTables; + readonly #context: Context; + /** Every id a write claims, in write order (documents created or copied included), for the id checks. */ + readonly #claims: { readonly id: number; readonly type: RecordType }[] = []; + readonly #rows: RowWrite[] = []; + readonly #actions = new Map(); + + /** + * Stages `writes` (rule 8: rows and contents encoded here, before the queue); `raise` makes room above each explicit + * id. A malformed batch (two content commands for one document, a second retirement) throws Durable's message. + */ + constructor( + writes: readonly StorageWrite[], + owner: string, + tables: CommitTables, + raise: (id: number) => void, + context: Context, + ) { + this.#owner = owner; + this.#tables = tables; + this.#context = context; + for (const write of writes) this.#stage(write, raise); + } + + /** + * Resolves the copies before the step (S11): each source read at its point and checked against the copy's record, + * entered as a creation carrying the source's value as its base. + */ + async resolveCopies(read: CopySource): Promise { + for (const [id, action] of this.#actions) { + const source = action.copy; + if (source === undefined) continue; + if (this.#actions.has(source.id)) throw new Error(`Document copy ${id} source is changed in the copy batch`); + try { + const stored = await read(source); + if (stored === undefined) throw new Error(`Fork source document ${source.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${source.id} does not match the copied record`); + } + action.base = { version: stored.version, value: stored.value, text: JSON.stringify(stored.value) }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + + async apply(step: Step): Promise { + const recorded: StoreWrite[] = []; + const claimed = await this.#checkIds(step); + const existing = await this.#checkDocuments(step); + for (const write of this.#rows) { + if (claimed.get(write.id) === undefined) recorded.push(await this.#claim(step, write.id, write.type)); + claimed.set(write.id, write.type); + const stamp = write.table === this.#tables.entries ? "commit_seq" : undefined; + recorded.push(await write.table.put(step, write.row, stamp)); + } + // Existing documents first: a retirement frees its address before a creation of this commit takes it. + for (const [id, record] of existing) recorded.push(...(await this.#change(step, record, this.#actions.get(id)!))); + for (const [id, action] of this.#actions) { + if (action.create === undefined) continue; + recorded.push(await this.#claim(step, id, "document")); + recorded.push(await this.#create(step, action.create, action.base!, action.retire)); + } + return recorded; + } + + /** Stages one write: a table row encoded, a document command into its action. */ + #stage(write: StorageWrite, raise: (id: number) => void): void { + const tables = this.#tables; + switch (write.type) { + case "conversation": + this.#row(write.value.id, write.type, tables.conversations, conversationRow(write.value), raise); + return; + case "entry": { + const { id, conversationId, head } = write.value; + this.#row(id, write.type, tables.entries, entryRow(write.value), raise); + // Its claim is the entry's: `apply` claims an id once. + if (head !== undefined) { + const row = tables.entryHeads.encode({ id, conversation_id: conversationId }); + this.#rows.push({ id, type: write.type, table: tables.entryHeads, row }); + } + return; + } + case "task": + this.#row(write.value.id, write.type, tables.tasks, taskRow(write.value), raise); + return; + case "submission": + this.#row(write.value.id, write.type, tables.submissions, submissionRow(write.value), raise); + return; + case "document.create": + case "document.copy": { + const { id } = write.record; + const action = this.#action(id); + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${id} has more than one content command`); + } + action.create = write.record; + if (write.type === "document.copy") action.copy = write.source; + else action.base = { ...write.content, text: JSON.stringify(write.content.value) }; + this.#claims.push({ id, type: "document" }); + raise(id); + return; + } + case "document.change": { + const action = this.#action(write.id); + // A creation's base is its content command (`SqliteStorage` counts it so too), in either order. + if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) { + throw new Error(`Document ${write.id} has more than one content command`); + } + const { content } = write; + action.content = { content, text: JSON.stringify(content.kind === "base" ? content.value : content.ops) }; + return; + } + case "document.retire": { + const action = this.#action(write.id); + if (action.retire) throw new Error(`Document ${write.id} is retired more than once`); + action.retire = true; + return; + } + } + } + + #row(id: Id, type: RowWrite["type"], table: OwnerTable, row: object, raise: (id: number) => void): void { + this.#claims.push({ id, type }); + this.#rows.push({ id, type, table, row: table.encode(row) }); + raise(id); + } + + #action(id: DocumentId): DocumentAction { + let action = this.#actions.get(id); + if (action === undefined) { + action = { retire: false }; + this.#actions.set(id, action); + } + return action; + } + + /** + * Durable's global id checks (`SqliteStorage.checkGlobalIds`): a conversation, entry or document id claimed by + * anything, or written twice; a task or submission id claimed by another type. Returns the stored types. + */ + async #checkIds(step: Step): Promise> { + const stored = new Map(); + const claimed = new Map(); + for (const { id, type } of this.#claims) { + let existing = stored.get(id); + if (!stored.has(id)) { + const row = await this.#tables.recordIds.get(step, [id], this.#context); + existing = row?.record_type as RecordType | undefined; + stored.set(id, existing); + } + const earlier = claimed.get(id); + if (type === "conversation" || type === "entry" || type === "document") { + if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`); + } else { + if (existing !== undefined && existing !== type) throw new Error(`ID ${id} already belongs to ${existing}`); + if (earlier !== undefined && earlier !== type) throw new Error(`ID ${id} is written as two record types`); + } + claimed.set(id, type); + } + const types = new Map(); + for (const [id, type] of stored) if (type !== undefined) types.set(id, type); + return types; + } + + /** + * Durable's document checks (`SqliteStorage.checkDocumentActions`): a change or retirement of an unknown or retired + * document, a delta on another version, more than one live incarnation per address after the commit. Returns the + * existing records the commit changes or retires, in action order. + */ + async #checkDocuments(step: Step): Promise> { + const existing = new Map(); + for (const [id, action] of this.#actions) { + if (action.create !== undefined) continue; + const stored = await readDocument(step, id); + const record = stored === undefined ? undefined : durableRecord(stored, this.#owner); + if (record === undefined) throw new Error(`Unknown document: ${id}`); + if (record.retiredAt !== undefined) throw new Error(`Document ${id} is retired`); + const content = action.content?.content; + if (content?.kind === "delta" && content.version !== stored!.version) { + throw new Error(`Document ${id} version transition requires a base`); + } + existing.set(id, stored!); + } + /** Per address a live creation takes: its live incarnations after the commit so far. */ + const live = new Map(); + for (const action of this.#actions.values()) { + if (action.create === undefined || action.retire) continue; + const address = addressOf(action.create, this.#owner); + const key = JSON.stringify([address.namespace, address.kind, address.key]); + let count = live.get(key); + if (count === undefined) { + // The incarnation live there now counts unless this commit retires it. + const current = await findDocument(step, address); + count = current === undefined || this.#actions.get(current.id as DocumentId)?.retire === true ? 0 : 1; + } + if (++count > 1) throw new Error("Document address already has a current incarnation"); + live.set(key, count); + } + return existing; + } + + /** Puts `id`'s `record_ids` row: its record type. */ + #claim(step: Step, id: number, type: RecordType): Promise { + const table = this.#tables.recordIds; + return table.put(step, table.encode({ id, record_type: type })); + } + + /** An existing document's content, then its retirement, as their writes. */ + async #change(step: Step, record: EngineRecord, action: DocumentAction): Promise { + const writes: StoreWrite[] = []; + let { revision } = record; + if (action.content !== undefined) { + const { content, text } = action.content; + const stored = engineContent(content); + await writeContent(step, record.id, record.history, revision, stored, text); + writes.push({ kind: "document.change", id: record.id, revision, content: stored }); + revision++; + } + if (action.retire) { + await retireRecord(step, record); + writes.push({ kind: "document.retire", id: record.id, revision }); + } + return writes; + } + + /** A creation or a resolved copy (retired at once for a create-plus-retire, an empty lifetime), as its write. */ + async #create(step: Step, create: DocumentCreate, base: StagedBase, retired: boolean): Promise { + const record = engineRecord(create, this.#owner, base.version, step.seq, retired); + await insertDocument(step, record, base.text); + const value = base.value as JsonValue; + return { kind: "document.create", record, base: { version: base.version, fingerprint: "", value } }; + } +} + +/** A Durable content as the engine stores it: its documents have no schema fingerprint. */ +function engineContent(content: DocumentContent): EngineContent { + return content.kind === "base" + ? { kind: "base", version: content.version, fingerprint: "", value: content.value } + : { kind: "delta", version: content.version, ops: content.ops }; +} diff --git a/packages/durable/src/storage/store/documents.ts b/packages/durable/src/storage/store/documents.ts new file mode 100644 index 000000000..d81f94d2d --- /dev/null +++ b/packages/durable/src/storage/store/documents.ts @@ -0,0 +1,143 @@ +// Durable's documents in the store's documents engine (slice 6b §1.5, R1): a Durable address as an engine address +// under the storage's owner name (its scope as the namespace, its kind and key JSON-encoded as Durable's indexed +// strings), a Durable record as an engine record and back (the Durable record in `meta`), and a document read at a +// point. + +import { + type DocumentEntry, + type DocumentHistory, + type DocumentAddress as EngineAddress, + type DocumentPoint as EnginePoint, + type DocumentRecord as EngineRecord, + loadContent, + readDocumentEntry, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { seqFromNumber } from "../../ids.ts"; +import type { + DocumentAddress, + DocumentCreate, + DocumentId, + DocumentPoint, + DocumentRecord, + JsonObject, + StoredDocument, +} from "../../types.ts"; +import { encodeIndexed } from "./tables.ts"; + +/** The namespaces a storage claims under its owner: Durable's scopes (Chord's namespaces never contain a `:`). */ +export const SCOPE_NAMESPACES = ["session", "conversation:", "task:"] as const; + +/** The engine namespace of a Durable scope: `session`, `conversation:`, `task:`. */ +export function namespaceOf(scope: DocumentRecord["scope"]): string { + switch (scope.kind) { + case "session": + return "session"; + case "conversation": + return `conversation:${scope.conversationId}`; + case "task": + return `task:${scope.taskId}`; + } +} + +/** The engine address of a Durable address under `owner`. */ +export function addressOf(address: DocumentAddress | DocumentCreate, owner: string): EngineAddress { + const at = { owner, namespace: namespaceOf(address.scope), kind: encodeIndexed(address.kind) }; + return address.key === undefined ? at : { ...at, key: encodeIndexed(address.key) }; +} + +/** What a record of `scope` keeps: session and task documents and latest conversation ones their latest state. */ +export function historyOf(record: DocumentCreate | DocumentRecord): DocumentHistory { + return record.scope.kind === "conversation" && record.history === "rewindable" ? "rewindable" : "latest"; +} + +/** + * The engine record of a Durable creation at `seq` (retired at once for an empty lifetime): the Durable record itself, + * its creation seq included, in `meta`, so a read of a live record parses one JSON text from one column, as + * `SqliteStorage`'s does (a retirement later sets only the engine's `retired_at`). + */ +export function engineRecord( + create: DocumentCreate, + owner: string, + version: number, + seq: number, + retired: boolean, +): EngineRecord { + return { + id: create.id, + ...addressOf(create, owner), + history: historyOf(create), + createdAt: seq, + ...(retired ? { retiredAt: seq } : {}), + meta: JSON.stringify({ ...create, createdAt: seq }), + revision: 1, + version, + fingerprint: "", + }; +} + +/** + * The Durable record of an engine record of this storage's owner and scopes, `undefined` for anyone else's (a Chord + * document, another owner's): its id is unknown to this storage. + */ +export function durableRecord(record: EngineRecord, owner: string): DocumentRecord | undefined { + const { meta } = record; + if (meta === undefined || !ownedHere(record, owner)) return undefined; + return recordOfEntry({ meta, retired_at: record.retiredAt ?? null }); +} + +/** The Durable record of an engine entry read at one of this storage's addresses or scopes (the record in `meta`). */ +export function recordOfEntry(entry: DocumentEntry): DocumentRecord { + // Written by `engineRecord`, parsed fresh: a retirement since is set on it in place. + const record = JSON.parse(entry.meta!) as { -readonly [K in keyof DocumentRecord]: DocumentRecord[K] }; + if (entry.retired_at !== undefined && entry.retired_at !== null) record.retiredAt = seqFromNumber(entry.retired_at); + return record as DocumentRecord; +} + +/** Whether a record is this storage's: `owner`'s, in one of Durable's scopes (not a Chord document of the owner). */ +function ownedHere(record: { readonly owner: string; readonly namespace: string }, owner: string): boolean { + const { namespace } = record; + if (record.owner !== owner) return false; + return namespace === "session" || namespace.startsWith("conversation:") || namespace.startsWith("task:"); +} + +/** Whether a document is alive at `at` (the half-open lifetime). */ +export function aliveAt(record: DocumentRecord, at: DocumentPoint): boolean { + if (at === "current") return record.retiredAt === undefined; + return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt); +} + +/** + * A document read at a point, as a tagged result: request errors are told after the guarded read (slice 6b §1.4), + * never thrown inside it. + */ +export type DocumentRead = + | { readonly kind: "found"; readonly document: StoredDocument | undefined } + | { readonly kind: "current_only"; readonly id: DocumentId }; + +/** + * Document `id` of `owner` at `at`: absent (unknown, someone else's, not alive at `at`), a record whose history does + * not reach back to `at`, or its value from the base at or below `at` and the deltas up to it. + */ +export async function readAt( + executor: Pick, + id: DocumentId, + at: DocumentPoint, + owner: string, +): Promise { + const entry = await readDocumentEntry(executor, id); + if (entry === undefined || entry.meta === null || !ownedHere(entry, owner)) { + return { kind: "found", document: undefined }; + } + const record = recordOfEntry(entry); + if (at !== "current" && entry.history !== "rewindable") return { kind: "current_only", id }; + if (!aliveAt(record, at)) return { kind: "found", document: undefined }; + const point: EnginePoint = at; + const loaded = await loadContent(executor, id, point); + // The engine stores what Durable's commits gave it: JSON objects. + const value = loaded.value as JsonObject; + return { + kind: "found", + document: { record, version: loaded.version, value, deltasSinceBase: loaded.deltasSinceBase }, + }; +} diff --git a/packages/durable/src/storage/store/index.ts b/packages/durable/src/storage/store/index.ts new file mode 100644 index 000000000..0ddf86cdc --- /dev/null +++ b/packages/durable/src/storage/store/index.ts @@ -0,0 +1 @@ +export { StoreStorage } from "./storage.ts"; diff --git a/packages/durable/src/storage/store/queries.ts b/packages/durable/src/storage/store/queries.ts new file mode 100644 index 000000000..8491270b7 --- /dev/null +++ b/packages/durable/src/storage/store/queries.ts @@ -0,0 +1,137 @@ +// StoreStorage's reads as the store's queries (slice 6b D2, §2.4), compiled once from the store's query vocabulary +// into SQL over the declared tables: no SQL is rendered per call, and a read selects only the record cell (by id, an +// entry's commit seq too; each column returned costs) and parses one JSON text per record, as `SqliteStorage`'s do. + +import type { Context } from "@earendil-works/chord"; +import type { Order, OwnerTable, WhereNode } from "@earendil-works/chord/next"; +import type { SqliteExecutor, SqliteValue } from "@earendil-works/chord/sqlite"; +import type { CommitTables } from "./commit.ts"; + +/** + * A stored row as a query selects it: the record cell, its version (selected only once the column has versions past + * 1), and an entry's commit seq. + */ +type RecordRow = { readonly record: string; readonly record$v?: number; readonly commit_seq?: number }; + +/** One condition of a read: a column, how it compares, its value; a call site keeps one order of them. */ +export type Condition = readonly [string, "eq" | "gt" | "lt" | "gte" | "lte", SqliteValue]; + +const ASCENDING: Order = { columns: ["id"], direction: "asc" }; +const DESCENDING: Order = { columns: ["id"], direction: "desc" }; + +/** The columns of a read of ids only. */ +const ID = ["id"]; + +/** A placeholder value: a query's shape binds its values at each call. */ +const ANY = 0; + +/** + * One table's compiled reads: by id (with the `byIdExtra` columns), and by shape (conditions in binding order, an + * order; the record only), each shape compiled at its first use. + */ +export class TableQueries { + readonly #table: OwnerTable; + /** The record column's read of a cell at a version past 1; `undefined` while the column is at version 1. */ + readonly #versioned: ((text: string, version: number, context: Context) => unknown) | undefined; + readonly #byId: string; + readonly #shapes = new Map(); + readonly #columns: readonly string[]; + + constructor(table: OwnerTable, byIdExtra: readonly string[] = []) { + this.#table = table; + const versioned = table.jsonVersion("record") > 1; + this.#versioned = versioned ? table.reader("record") : undefined; + this.#columns = versioned ? ["record", "record$v"] : ["record"]; + this.#byId = table.prepareGet([...this.#columns, ...byIdExtra]); + } + + /** The row with `id`, `undefined` when absent. */ + byId(executor: Pick, id: number): Promise { + return executor.get(this.#byId, id); + } + + /** Up to `limit` rows whose `conditions` hold, by id in `direction`. */ + select( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + limit: number, + ): Promise { + const sql = this.#shape(conditions, direction, "many"); + return executor.all(sql, ...conditions.map(([, , value]) => value), limit); + } + + /** The first row whose `conditions` hold, by id in `direction`; `undefined` when none does. */ + first( + executor: Pick, + conditions: readonly Condition[], + direction: "ascending" | "descending", + ): Promise { + const sql = this.#shape(conditions, direction, "first"); + return executor.get(sql, ...conditions.map(([, , value]) => value)); + } + + /** A row's record, read as its column's (version checked, migrated on access). */ + record(row: RecordRow, context: Context): T { + const versioned = this.#versioned; + // The cell holds a record of this table the storage wrote; at version 1 it is that JSON (`OwnerTable.reader`). + return (versioned === undefined ? JSON.parse(row.record) : versioned(row.record, row.record$v!, context)) as T; + } + + /** Appends the records of `rows` to `into`, in order, each read as `record` reads it (one check per call). */ + records(rows: readonly RecordRow[], into: T[], context: Context): void { + const versioned = this.#versioned; + if (versioned === undefined) { + for (const row of rows) into.push(JSON.parse(row.record) as T); + return; + } + for (const row of rows) into.push(versioned(row.record, row.record$v!, context) as T); + } + + /** The compiled read of a shape: its conditions' columns and comparisons, the direction, one row or many. */ + #shape(conditions: readonly Condition[], direction: "ascending" | "descending", rows: "first" | "many"): string { + let key = `${rows} ${direction}`; + for (const [column, condition] of conditions) key += ` ${column} ${condition}`; + let sql = this.#shapes.get(key); + if (sql !== undefined) return sql; + // One comparison per condition: the vocabulary's node of that name, its value a placeholder. + const nodes = conditions.map(([column, condition]) => ({ [condition]: [column, ANY] }) as unknown as WhereNode); + const where: WhereNode | undefined = nodes.length === 0 ? undefined : { and: nodes }; + const order = direction === "ascending" ? ASCENDING : DESCENDING; + sql = + rows === "first" + ? this.#table.prepareFirst(this.#columns, where, order) + : this.#table.prepare(this.#columns, where, order); + this.#shapes.set(key, sql); + return sql; + } +} + +/** The compiled reads of the storage's tables. */ +export type StorageQueries = { + readonly conversations: TableQueries; + readonly entries: TableQueries; + /** The id of a conversation's newest entry with a head marker: any, or at or before an id (bound second). */ + readonly latestHead: { readonly any: string; readonly atOrBefore: string }; + readonly tasks: TableQueries; + readonly submissions: TableQueries; +}; + +/** The reads of `tables`. */ +export function storageQueries(tables: CommitTables): StorageQueries { + const heads = tables.entryHeads; + return { + conversations: new TableQueries(tables.conversations), + entries: new TableQueries(tables.entries, ["commit_seq"]), + latestHead: { + any: heads.prepareFirst(ID, { eq: ["conversation_id", ANY] }, DESCENDING), + atOrBefore: heads.prepareFirst( + ID, + { and: [{ eq: ["conversation_id", ANY] }, { lte: ["id", ANY] }] }, + DESCENDING, + ), + }, + tasks: new TableQueries(tables.tasks), + submissions: new TableQueries(tables.submissions), + }; +} diff --git a/packages/durable/src/storage/store/storage.ts b/packages/durable/src/storage/store/storage.ts new file mode 100644 index 000000000..554979b89 --- /dev/null +++ b/packages/durable/src/storage/store/storage.ts @@ -0,0 +1,557 @@ +// Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): opening under an owner name (claims, tables, the +// allocator), commits as one composite write of the store's commit step, and the reads, each one guarded read of the +// store (several queries at one state in a read transaction), its request errors in a tagged result told after it. + +import type { Context, JsonValue } from "@earendil-works/chord"; +import { + type ChordStore, + type Claimant, + findDocumentEntry, + listDocumentEntries, + type Store, + StoreOwnerTables, +} from "@earendil-works/chord/next"; +import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { StorageRequestError } from "../../errors.ts"; +import { idFromNumber, seqFromNumber } from "../../ids.ts"; +import type { + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentCopySource, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + ScanOrder, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../../types.ts"; +import { nextCursor, type ScanStart, scanStart } from "../scan.ts"; +import { type CommitTables, DurableCommit } from "./commit.ts"; +import { addressOf, type DocumentRead, namespaceOf, readAt, recordOfEntry, SCOPE_NAMESPACES } from "./documents.ts"; +import { type Condition, type StorageQueries, storageQueries, type TableQueries } from "./queries.ts"; +import { Conversations, Entries, EntryHeads, encodeIndexed, RecordIds, Submissions, Tasks } from "./tables.ts"; + +type StoredTask = TaskRecord; +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** + * A guarded read's result: its data, or its request error, which is thrown after the guarded read (slice 6b §1.4), + * never inside it. + */ +type Read = { readonly kind: "found"; readonly value: T } | { readonly kind: "refused"; readonly message: string }; + +/** The id condition of a table scan from `start`. */ +function scanFrom(start: ScanStart): Condition { + return start.order === "ascending" + ? ["id", "gt", start.after ?? -1] + : ["id", "lt", start.after ?? Number.MAX_SAFE_INTEGER]; +} + +function page }>( + values: readonly T[], + limit: number, + order: ScanOrder, +): Page { + const items = values.slice(0, limit); + if (values.length <= limit) return { items }; + return { items, next: nextCursor(items.at(-1)!.id, order) }; +} + +function cursorId(cursor: Cursor | undefined): number | undefined { + const after = cursor?.after; + if (after === undefined) return undefined; + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); + return after; +} + +/** + * Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): its records as declared tables under an owner name, its + * documents in the store's documents engine, its ids from the store's one allocator, its commits as one composite + * write of the store's commit step, adopted in the host's batch. Several may share a store under distinct owner names + * (claims); the app owns the store, and `close` leaves it open. + */ +export class StoreStorage implements Storage { + readonly #store: ChordStore; + readonly #owner: string; + readonly #claimant: Claimant; + readonly #tables: CommitTables; + readonly #queries: StorageQueries; + #closed = false; + #closing: Promise | undefined; + /** Calls underway, which `close` waits for before it releases the claims (`#drained` once they reach 0). */ + #underway = 0; + #drained: (() => void) | undefined; + + private constructor(store: ChordStore, owner: string, claimant: Claimant, tables: CommitTables) { + this.#store = store; + this.#owner = owner; + this.#claimant = claimant; + this.#tables = tables; + this.#queries = storageQueries(tables); + } + + /** + * Opens Durable's storage in `store` under `owner` (a lowercase `[a-z0-9@/._~-]` name): claims its tables and its + * document namespaces (a second claim is refused `invalid/args`), creates or migrates its tables, and raises the + * store's allocator above every id it holds (one index read). A failure releases what it claimed. + */ + static async open(store: Store, owner: string, context: Context): Promise { + // Chord's own: `openStore` or `importStore` made it. + const chord = store as ChordStore; + const claimant: Claimant = { label: `StoreStorage(${owner})` }; + try { + const declared = new StoreOwnerTables(chord, owner, claimant); + const tables: CommitTables = { + recordIds: declared.declare(RecordIds), + conversations: declared.declare(Conversations), + entries: declared.declare(Entries), + entryHeads: declared.declare(EntryHeads), + tasks: declared.declare(Tasks), + submissions: declared.declare(Submissions), + }; + for (const namespace of SCOPE_NAMESPACES) chord.claims.documents(owner, namespace, claimant); + await declared.prepare(context); + const highest = await chord.read( + (executor) => + executor.get<{ readonly id: number | null }>( + `SELECT max("id") AS id FROM "${tables.recordIds.sql.table}"`, + ), + context, + ); + if (highest?.id != null) chord.ids.raise(highest.id); + return new StoreStorage(chord, owner, claimant, tables); + } catch (error) { + chord.claims.release(claimant); + throw error; + } + } + + commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { + return this.#call(async () => { + const store = this.#store; + const commit = new DurableCommit(writes, this.#owner, this.#tables, (id) => store.ids.raise(id), context); + await commit.resolveCopies((source) => this.#copySource(source, context)); + // An empty commit stages nothing: it takes no seq, adopted at `lastSeq`. + const staged = writes.length === 0 ? [] : [commit]; + return seqFromNumber(await store.commitInBatch(staged, (seq) => adopt(seqFromNumber(seq)), context)); + }); + } + + mintId>(): Promise { + return this.#call(async () => idFromNumber(this.#store.mintId())); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#read((executor) => this.#conversation(executor, id, context), context); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.ownerConversationId !== undefined) { + conditions.push(["owner_conversation_id", "eq", query.ownerConversationId]); + } + if (query.ownerTaskId !== undefined) conditions.push(["owner_task_id", "eq", query.ownerTaskId]); + return this.#scan(this.#queries.conversations, conditions, start, limit, context); + }); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + idOrConversationId: EntryId | ConversationId, + idOrContext: EntryId | Context, + maybeContext?: Context, + ): Promise { + if (maybeContext === undefined) { + const context = idOrContext as Context; + return this.#read((executor) => this.#entry(executor, idOrConversationId as EntryId, context), context); + } + const context = maybeContext; + const id = typeof idOrContext === "number" ? idFromNumber(idOrContext) : undefined; + return this.#reading(() => { + if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID"); + const conversationId = idOrConversationId as ConversationId; + return this.#store + .read((executor) => this.#visibleEntry(executor, conversationId, id, context), context) + .then(told); + }); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#reading(() => + this.#store + .read((executor) => this.#latestHeadMarker(executor, conversationId, atOrBeforeEntryId, context), context) + .then(told), + ); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const { order, after } = scanStart(query.order, cursor, "descending"); + const read = + order === "ascending" + ? (executor: SqliteExecutor) => this.#entriesAscending(executor, query, limit, after, context) + : (executor: SqliteExecutor) => this.#entriesDescending(executor, query, limit, after, context); + return this.#store.read(read, context).then(told); + }); + } + + task(id: TaskId, context: Context): Promise { + const tasks = this.#queries.tasks; + return this.#read(async (executor) => { + const row = await tasks.byId(executor, id); + return row === undefined ? undefined : tasks.record(row, context); + }, context); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.kind !== undefined) conditions.push(["kind", "eq", encodeIndexed(query.kind)]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + if (query.abortRequested !== undefined) + conditions.push(["abort_requested", "eq", query.abortRequested ? 1 : 0]); + if (query.background !== undefined) conditions.push(["background", "eq", query.background ? 1 : 0]); + return this.#scan(this.#queries.tasks, conditions, start, limit, context); + }); + } + + submission(id: SubmissionId, context: Context): Promise { + const submissions = this.#queries.submissions; + return this.#read(async (executor) => { + const row = await submissions.byId(executor, id); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const start = scanStart(query.order, cursor, "ascending"); + const conditions: Condition[] = [scanFrom(start)]; + if (query.conversationId !== undefined) conditions.push(["conversation_id", "eq", query.conversationId]); + if (query.status !== undefined) conditions.push(["status", "eq", query.status]); + return this.#scan(this.#queries.submissions, conditions, start, limit, context); + }); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + const submissions = this.#queries.submissions; + const conditions: Condition[] = [ + ["conversation_id", "eq", conversationId], + ["request_id", "eq", encodeIndexed(requestId)], + ]; + return this.#read(async (executor) => { + const row = await submissions.first(executor, conditions, "ascending"); + return row === undefined ? undefined : submissions.record(row, context); + }, context); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#read(async (executor) => { + const entry = await findDocumentEntry(executor, addressOf(address, this.#owner), at); + return entry === undefined ? undefined : recordOfEntry(entry); + }, context); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + // The record and its revisions must come from one committed state (S4). + return this.#reading(() => + this.#store.readTransaction((executor) => readAt(executor, id, at, this.#owner), context).then(documentOf), + ); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#reading(() => { + const after = cursorId(cursor) ?? -1; + const scope = { + owner: this.#owner, + namespace: namespaceOf(query.scope), + ...(query.kind === undefined ? {} : { kind: encodeIndexed(query.kind) }), + }; + return this.#store + .read((executor) => listDocumentEntries(executor, scope, query.at, after, limit + 1), context) + .then((found) => page(found.map(recordOfEntry), limit, "ascending")); + }); + } + + /** + * Refuses new calls, waits for its commits underway, then releases its claims; the store stays open (the app's), + * and finishes the reads underway, which change nothing. + */ + close(_context: Context): Promise { + if (this.#closing === undefined) { + this.#closed = true; + this.#closing = (async () => { + if (this.#underway > 0) { + await new Promise((resolve) => { + this.#drained = resolve; + }); + } + this.#store.claims.release(this.#claimant); + })(); + } + return this.#closing; + } + + /** Runs one commit or mint of this storage: refused once it closed, counted until it settles (`close` waits). */ + async #call(run: () => Promise): Promise { + if (this.#closed) throw new Error("StoreStorage is closed"); + this.#underway++; + try { + return await run(); + } finally { + if (--this.#underway === 0) this.#drained?.(); + } + } + + /** + * A call that is one guarded read of the store: refused once closed; not waited for by `close` (a read changes + * nothing, and the store, which stays open, finishes it). + */ + #read(read: (executor: SqliteExecutor) => Promise, context: Context): Promise { + return this.#reading(() => this.#store.read(read, context)); + } + + /** + * A read call: `read` prepares its guarded read (a request error thrown then, before it) and returns it. Refused + * once closed; one promise per call besides the guard's (no async wrapper). + */ + #reading(read: () => Promise): Promise { + try { + if (this.#closed) throw new Error("StoreStorage is closed"); + return read(); + } catch (error) { + return Promise.reject(error); + } + } + + async #conversation( + executor: SqliteExecutor, + id: ConversationId, + context: Context, + ): Promise { + const conversations = this.#queries.conversations; + const row = await conversations.byId(executor, id); + return row === undefined ? undefined : conversations.record(row, context); + } + + #entry(executor: SqliteExecutor, id: EntryId, context: Context): Promise { + const entries = this.#queries.entries; + return entries.byId(executor, id).then((row) => { + if (row === undefined) return undefined; + return { entry: entries.record(row, context), commitSeq: seqFromNumber(row.commit_seq!) }; + }); + } + + /** Entry `id` when visible through `conversationId`'s fork chain (`SqliteStorage.readEntry`). */ + async #visibleEntry( + executor: SqliteExecutor, + conversationId: ConversationId, + id: EntryId, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + const stored = await this.#entry(executor, id, context); + if (stored === undefined) return NOTHING_FOUND; + let upper = Number.POSITIVE_INFINITY; + while (conversation.id !== stored.entry.conversationId) { + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return stored.entry.id > upper ? NOTHING_FOUND : { kind: "found", value: stored }; + } + + async #latestHeadMarker( + executor: SqliteExecutor, + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise> { + let conversation = await this.#conversation(executor, conversationId, context); + if (conversation === undefined) return unknownConversation(conversationId); + let upper: number | undefined = atOrBeforeEntryId; + while (true) { + const { latestHead } = this.#queries; + const head = + upper === undefined + ? await executor.get<{ readonly id: number }>(latestHead.any, conversation.id) + : await executor.get<{ readonly id: number }>(latestHead.atOrBefore, conversation.id, upper); + if (head !== undefined) { + // Written with its entry, in the same commit. + const row = (await this.#queries.entries.byId(executor, head.id))!; + const value = this.#queries.entries.record(row, context); + return { kind: "found", value }; + } + if (conversation.parent === undefined) return NOTHING_FOUND; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + } + + /** Newest first: each conversation of the fork chain up to its fork point (`SqliteStorage.readEntries`). */ + async #entriesDescending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1); + const values: EntryRecord[] = []; + while (true) { + const range = this.#range(conversation.id, query.minEntryId, upper); + const rows = await this.#queries.entries.select(executor, range, "descending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit || conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + return { kind: "found", value: page(values, limit, "descending") }; + } + + /** Oldest first: the fork chain's segments from the root forward, each up to its fork point. */ + async #entriesAscending( + executor: SqliteExecutor, + query: EntryQuery, + limit: number, + after: number | undefined, + context: Context, + ): Promise>> { + const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = []; + let conversation = await this.#conversation(executor, query.conversationId, context); + if (conversation === undefined) return unknownConversation(query.conversationId); + let upper: number | undefined = query.maxEntryId; + while (true) { + segments.push({ conversationId: conversation.id, upper }); + if (conversation.parent === undefined) break; + upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at); + if (query.minEntryId !== undefined && upper < query.minEntryId) break; + conversation = (await this.#conversation(executor, conversation.parent.conversationId, context))!; + } + let lower: number | undefined = query.minEntryId; + if (after !== undefined) lower = Math.max(lower ?? after + 1, after + 1); + const values: EntryRecord[] = []; + for (const segment of segments.reverse()) { + const range = this.#range(segment.conversationId, lower, segment.upper); + const rows = await this.#queries.entries.select(executor, range, "ascending", limit + 1 - values.length); + this.#queries.entries.records(rows, values, context); + if (values.length > limit) break; + } + return { kind: "found", value: page(values, limit, "ascending") }; + } + + /** One conversation's entries with ids from `lower` to `upper` (inclusive, each optional). */ + #range(conversationId: ConversationId, lower: number | undefined, upper: number | undefined): Condition[] { + const conditions: Condition[] = [["conversation_id", "eq", conversationId]]; + if (lower !== undefined) conditions.push(["id", "gte", lower]); + if (upper !== undefined) conditions.push(["id", "lte", upper]); + return conditions; + } + + /** A page of a table scan: up to `limit` records of `queries`' table whose `conditions` hold, from `start`. */ + #scan }>( + queries: TableQueries, + conditions: readonly Condition[], + start: ScanStart, + limit: number, + context: Context, + ): Promise> { + return this.#store + .read((executor) => queries.select(executor, conditions, start.order, limit + 1), context) + .then((rows) => { + const values: T[] = []; + queries.records(rows, values, context); + return page(values, limit, start.order); + }); + } + + /** A copy's source read at its point, before the step (S11); a request error is thrown here, outside the read. */ + async #copySource(source: DocumentCopySource, context: Context) { + const read = await this.#store.readTransaction( + (executor) => readAt(executor, source.id, source.at, this.#owner), + context, + ); + return documentOf(read); + } +} + +/** A read that found nothing. */ +const NOTHING_FOUND: Read = { kind: "found", value: undefined }; + +function unknownConversation(id: ConversationId): Read { + return { kind: "refused", message: `Unknown conversation: ${id}` }; +} + +/** A read's data, its request error thrown now that the guarded read returned (slice 6b §1.4). */ +function told(read: Read): T { + if (read.kind === "refused") throw new StorageRequestError(read.message); + return read.value; +} + +/** A document read's result, a record that keeps no history at the point thrown as Durable's request error. */ +function documentOf(read: DocumentRead): StoredDocument | undefined { + if (read.kind === "current_only") + throw new StorageRequestError(`Document ${read.id} does not retain historical content`); + return read.document; +} diff --git a/packages/durable/src/storage/store/tables.ts b/packages/durable/src/storage/store/tables.ts new file mode 100644 index 000000000..49523292d --- /dev/null +++ b/packages/durable/src/storage/store/tables.ts @@ -0,0 +1,122 @@ +// Durable's records as declared tables of the store (slice 6b D2, §2.2): every row carries its record's indexed columns +// and the whole record as one `json(...)` cell. Indexed strings that must stay lossless (`kind`, `request_id`) hold +// Durable's JSON encoding; index columns are never `opt`, so absent values take sentinels no encoded value can be +// (`0` for an absent owner, which is never an id; `""` for an absent request id: an encoded string starts with `"`). + +import type { JsonValue } from "@earendil-works/chord"; +import { defineTable, int, json, str, unknown } from "@earendil-works/chord/next"; +import type { ConversationRecord, EntryRecord, SubmissionRecord, TaskRecord } from "../../types.ts"; + +/** A record as one `json(...)` cell. */ +const record = () => json(unknown(), { version: 1 }); + +/** Which record type owns an id: Durable's one namespace across its five record types. */ +export const RecordIds = defineTable("record_ids", { + version: 1, + columns: { id: int(), record_type: str() }, + key: ["id"], +}); + +export const Conversations = defineTable("conversations", { + version: 1, + columns: { id: int(), owner_conversation_id: int(), owner_task_id: int(), record: record() }, + key: ["id"], + indexes: { by_owner_conversation: ["owner_conversation_id"], by_owner_task: ["owner_task_id"] }, +}); + +export const Entries = defineTable("entries", { + version: 1, + columns: { id: int(), conversation_id: int(), commit_seq: int(), record: record() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +/** + * The entries with a head marker, by conversation: what `SqliteStorage`'s partial index on `head IS NOT NULL` is, + * as rows written only with such an entry, so a commit of an entry without one updates no second index. + */ +export const EntryHeads = defineTable("entry_heads", { + version: 1, + columns: { id: int(), conversation_id: int() }, + key: ["id"], + indexes: { by_conversation: ["conversation_id"] }, +}); + +export const Tasks = defineTable("tasks", { + version: 1, + columns: { + id: int(), + conversation_id: int(), + kind: str(), + status: str(), + abort_requested: int(), + background: int(), + record: record(), + }, + key: ["id"], + indexes: { + by_status: ["status"], + by_conversation: ["conversation_id"], + by_kind: ["kind"], + by_abort_requested: ["abort_requested"], + by_background: ["background"], + }, +}); + +export const Submissions = defineTable("submissions", { + version: 1, + columns: { id: int(), conversation_id: int(), request_id: str(), status: str(), record: record() }, + key: ["id"], + indexes: { + by_request: ["conversation_id", "request_id"], + by_conversation: ["conversation_id"], + by_status: ["status"], + }, +}); + +/** Durable's JSON encoding of an indexed string: lossless whatever the string (lone surrogates, NUL). */ +export const encodeIndexed = (value: string): string => JSON.stringify(value); + +/** The row of a conversation record. */ +export function conversationRow(value: ConversationRecord): object { + return { + id: value.id, + owner_conversation_id: value.owner?.conversationId ?? 0, + owner_task_id: value.owner?.taskId ?? 0, + record: value, + }; +} + +/** The row of an entry record; `commit_seq` is stamped by the step. */ +export function entryRow(value: EntryRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + commit_seq: 0, + record: value, + }; +} + +/** The row of a task record. */ +export function taskRow(value: TaskRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + kind: encodeIndexed(value.kind), + status: value.state.status, + abort_requested: value.abortRequested ? 1 : 0, + background: value.background ? 1 : 0, + record: value, + }; +} + +/** The row of a submission record. */ +export function submissionRow(value: SubmissionRecord): object { + return { + id: value.id, + conversation_id: value.conversationId, + request_id: value.requestId === undefined ? "" : encodeIndexed(value.requestId), + status: value.status, + record: value, + }; +} diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 2797c0462..91b9cf057 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -1646,6 +1646,30 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, ), + createCase( + options, + "refuses a creation and a change of one document in one commit, in either order", + async (storage) => { + await createRoot(storage); + for (const order of ["create first", "change first"] as const) { + const id = await storage.mintId(); + const create: StorageWrite = { + type: "document.create", + record: { id, kind: "twice", key: order, scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { x: 1 } }, + }; + const change: StorageWrite = { + type: "document.change", + id, + content: { kind: "delta", version: 1, ops: [["s", ["x"], 2]] }, + }; + const writes = order === "create first" ? [create, change] : [change, create]; + await expect(storage.commit(writes, adopt, context)).rejects.toThrow("more than one content command"); + expect(await storage.document(id, "current", context)).toBeUndefined(); + } + }, + ), + createCase( options, "rolls back record tables and secondary indexes when a document command fails", diff --git a/packages/durable/test/conversation-view-bench.ts b/packages/durable/test/conversation-view-bench.ts index d9096293f..e3b781850 100644 --- a/packages/durable/test/conversation-view-bench.ts +++ b/packages/durable/test/conversation-view-bench.ts @@ -3,7 +3,8 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/conversation-view-bench.ts * - * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap: + * Scenarios, each in its own child process so heap numbers are its own, on SQLite so history stays out of the heap + * (`SqliteStorage`; with `PI_DURABLE_BENCH_STORAGE=store`, Durable's storage on Chord's store, `StoreStorage`): * - `start`: acquire a watch of a conversation of N entries (the mount is built every time; the watch is stopped). * - `append`: K started watches; commit entries one per commit, or in bursts of 50 per commit, and wait for delivery. * - `resume`: a watch with `from` 20 entries back, as a client that reconnects after a short drop. @@ -22,6 +23,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { type Conversation, createRegistry, type EntryId, Harness, LiveDoc, type Storage } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; type Scenario = | { kind: "start"; entries: number } @@ -114,7 +116,11 @@ async function run(scenario: Scenario): Promise> { const directory = mkdtempSync(join(tmpdir(), "pi-durable-view-bench-")); let storage: Storage | undefined; try { - storage = await openNodeSqliteStorage(join(directory, "session.sqlite")); + const path = join(directory, "session.sqlite"); + storage = + process.env.PI_DURABLE_BENCH_STORAGE === "store" + ? await openStoreStorage(path) + : await openNodeSqliteStorage(path); const models = createModels(); const harness = await Harness.open(storage, { models, registry: createRegistry() }, context); const root = await harness.root(context); diff --git a/packages/durable/test/examples/35-durable-on-the-store.ts b/packages/durable/test/examples/35-durable-on-the-store.ts new file mode 100644 index 000000000..958a9d1b4 --- /dev/null +++ b/packages/durable/test/examples/35-durable-on-the-store.ts @@ -0,0 +1,108 @@ +// Durable on the store: a session's worker keeps everything in one store. Durable's Session (a Harness) opens +// `StoreStorage` over the host's store under its object's type name, next to a Chord object's own table and documents; +// both commit through the store's one commit queue (one seq each), one id allocator numbers them all, and one export +// carries both. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/35-durable-on-the-store.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + createHost, + defineFacet, + defineTable, + local, + method, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Notes = defineTable("notes", { version: 1, columns: { id: str(), text: str() }, key: ["id"] }); + +// The harness object: local with empty params, so it lives as long as the host and may persist (P3). +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { note: method(str()), say: method(str()) }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + // Its own table, under its type name ... + const notes = self.table(Notes); + // ... and Durable's records under the same owner name: claims per table and per document namespace. + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + return { + note: async (text, ctx) => { + await self.tx(async (tx) => tx.put(notes, { id: `n${text.length}`, text }), ctx); + return null; + }, + say: async (text, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind: "example.said", data: { text } } }, ctx); + return null; + }, + }; + }); +}); + +const directory = await mkdtemp(join(tmpdir(), "chord-example-26-")); +const path = join(directory, "session.sqlite"); +// 1. The worker's start code opens the store and gives it to its host. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 2. Chord's commit and Durable's commit, each one seq of the same store. +const before = store.lastSeq; +await chat.note("hello", context); +await chat.say("hello, Durable", context); +console.log("seqs taken:", store.lastSeq - before); + +// 3. One archive holds both: Chord's table, Durable's tables and documents, all under the owner name. +const owners = new Map(); +for await (const part of store.export(context)) { + const owner = + part.kind === "table" || part.kind === "put" + ? part.table.split(":")[0] + : part.kind === "document.create" + ? part.record.owner + : undefined; + if (owner !== undefined) owners.set(`${owner} ${part.kind}`, (owners.get(`${owner} ${part.kind}`) ?? 0) + 1); + if (part.kind === "manifest") console.log("manifest:", { lastSeq: part.lastSeq, nextId: part.nextId }); +} +console.log("parts by owner and kind:", Object.fromEntries(owners)); + +// 4. The app closes the host, then the store; a reopen finds both. +await host.close(context); +await store.close(context); +const reopened = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); +const storage = await StoreStorage.open(reopened, Chat.name, context); +const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); +console.log( + "Durable entries after reopen:", + page.items.map((entry) => entry.kind), +); +await storage.close(context); +await reopened.close(context); +await rm(directory, { recursive: true, force: true }); diff --git a/packages/durable/test/examples/36-durable-views.ts b/packages/durable/test/examples/36-durable-views.ts new file mode 100644 index 000000000..2f2ace488 --- /dev/null +++ b/packages/durable/test/examples/36-durable-views.ts @@ -0,0 +1,91 @@ +// Durable's views as Chord sources: `self.follow(attach)` turns Durable's synchronous observers (the conversation view, +// the task graph) into watchables, members and derive sources of the object that opened Durable. With Durable on the +// host's store, Durable adopts its commits inside the store's commit step, in the host's batch: a derive over two views +// of one commit computes once, and a watcher gets one frame per commit. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/36-durable-views.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + int, + local, + method, + obj, + openStore, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { createRegistry, Harness, ROOT_CONVERSATION_ID } from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +const Chat = local("example.chat", { + version: 1, + params: {}, + members: { + // What a UI shows: the kinds of the transcript's entries and the number of tasks, from two views. + summary: obj({ kinds: arr(str()), tasks: int() }), + write: method(str()), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + // 1. Follow two of Durable's views; they attach once the natives opened. + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx, { min: 50 }), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + // 2. A derive over both: one compute per Durable commit. + const summary = self.derive([view, graph], ([v, g]) => ({ + kinds: v.entries.map((entry) => entry.kind), + tasks: Object.keys(g.tasks).length, + })); + return { + summary, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); +}); + +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const host = createHost({ store, onError: (error) => console.log("onError", error.kind) }); +await host.load([{ name: "@example/core", facet }], context); +const chat = await host.get(Chat, {}, context); + +// 3. Watch the member: one frame per Durable commit, with its ops. +const watch = await chat.summary.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log("frame:", call.value.kinds, JSON.stringify(call.ops)); +}); +await chat.write("example.first", context); +await chat.write("example.second", context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("member value:", chat.summary.value); + +// 4. Closing the host detaches the observers; the store is the app's. +await watch.stop(); +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/storage-memory.ts b/packages/durable/test/storage-memory.ts index 0e105fdc8..49828a4b0 100644 --- a/packages/durable/test/storage-memory.ts +++ b/packages/durable/test/storage-memory.ts @@ -17,8 +17,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; type MemorySnapshot = { @@ -125,11 +126,13 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma storage = new MemoryStorage(); } else { directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-memory-`)); - path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); storage = backend === "sqlite" ? await openNodeSqliteStorage(path) - : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); + : backend === "store" + ? await openStoreStorage(path) + : await openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } try { @@ -151,7 +154,7 @@ async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchma collectGarbage(); const postRead = snapshot(); let disk: DiskFootprint | undefined; - if (path !== undefined && backend === "sqlite") { + if (path !== undefined && backend !== "jsonl") { disk = { ...sqliteMetrics(path), mainBytes: await fileSize(path), diff --git a/packages/durable/test/storage.bench.ts b/packages/durable/test/storage.bench.ts index f24e915eb..68fdb06a3 100644 --- a/packages/durable/test/storage.bench.ts +++ b/packages/durable/test/storage.bench.ts @@ -14,8 +14,9 @@ import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import type { EntryId, Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; -const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const; +const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "store", "jsonl"] as const; type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number]; const READ_OPTIONS = { time: 300, iterations: 10, warmupTime: 75, warmupIterations: 3 } as const; @@ -38,9 +39,10 @@ async function createFixture(backend: StorageBenchmarkBackend): Promise } const directory = await mkdtemp(join(tmpdir(), "pi-durable-benchmark-")); directories.push(directory); - if (backend === "sqlite") { + if (backend === "sqlite" || backend === "store") { const path = join(directory, "storage.sqlite"); - const fixture = { backend, storage: await openNodeSqliteStorage(path) } satisfies Fixture; + const storage = backend === "sqlite" ? await openNodeSqliteStorage(path) : await openStoreStorage(path); + const fixture = { backend, storage } satisfies Fixture; fixtures.push(fixture); return fixture; } @@ -123,11 +125,12 @@ type ReopenFixture = { }; async function openPersistentStorage(backend: PersistentBackend, path: string): Promise { + if (backend === "store") return openStoreStorage(path); return backend === "sqlite" ? openNodeSqliteStorage(path) : openNodeJsonlStorage(path, BACKGROUND_CONTEXT); } async function copyPersistentStorage(backend: PersistentBackend, source: string, destination: string): Promise { - if (backend === "sqlite") await copyFile(source, destination); + if (backend !== "jsonl") await copyFile(source, destination); else await cp(source, destination, { recursive: true }); } @@ -136,7 +139,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { if (backend === "memory") continue; const seedDirectory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-benchmark-`)); directories.push(seedDirectory); - const seedPath = join(seedDirectory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const seedPath = join(seedDirectory, backend === "jsonl" ? "storage" : "storage.sqlite"); const seed = await openPersistentStorage(backend, seedPath); const dataset = await seedStorageBenchmark(seed); await seed.close(BACKGROUND_CONTEXT); @@ -144,7 +147,7 @@ for (const backend of STORAGE_BENCHMARK_BACKENDS) { Array.from({ length: REOPEN_OPTIONS.iterations + REOPEN_OPTIONS.warmupIterations }, async () => { const directory = await mkdtemp(join(tmpdir(), `pi-durable-${backend}-reopen-sample-`)); directories.push(directory); - const path = join(directory, backend === "sqlite" ? "storage.sqlite" : "storage"); + const path = join(directory, backend === "jsonl" ? "storage" : "storage.sqlite"); await copyPersistentStorage(backend, seedPath, path); return path; }), diff --git a/packages/durable/test/store-storage-node.ts b/packages/durable/test/store-storage-node.ts new file mode 100644 index 000000000..3dff775d4 --- /dev/null +++ b/packages/durable/test/store-storage-node.ts @@ -0,0 +1,23 @@ +// What `vitest.store.config.ts` puts in the place of `../src/storage/sqlite/node.ts`: the same exports, with +// `openNodeSqliteStorage(path)` opening Durable's storage on Chord's store over that file (`StoreStorage`), so the +// harness test files that take a SQLite storage run unchanged on the store. + +import type { NodeSqliteDatabaseOptions } from "@earendil-works/chord/sqlite/node"; +import type { SqliteStorageOptions } from "../src/storage/sqlite/index.ts"; +import type { Storage } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +export { + NodeSqliteDatabase, + type NodeSqliteDatabaseOptions, + openNodeSqliteDatabase, +} from "@earendil-works/chord/sqlite/node"; + +/** Node SQLite connection settings plus durable storage options, as the original takes them. */ +export type NodeSqliteStorageOptions = NodeSqliteDatabaseOptions & SqliteStorageOptions; + +/** A `StoreStorage` on a store over the file at `path`; a table prefix becomes nothing (the owner name separates). */ +export function openNodeSqliteStorage(path: string, options: NodeSqliteStorageOptions = {}): Promise { + const { prefix: _prefix, ...database } = options; + return openStoreStorage(path, database); +} diff --git a/packages/durable/test/store-storage-support.ts b/packages/durable/test/store-storage-support.ts new file mode 100644 index 000000000..1143d6eaf --- /dev/null +++ b/packages/durable/test/store-storage-support.ts @@ -0,0 +1,46 @@ +// StoreStorage over a SQLite file for tests: a store opened on the file, Durable's storage in it under a test owner +// name, and a close that closes the store too (the app's part: in tests the storage is the store's only owner). + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { openStore, sqliteBackend } from "@earendil-works/chord/next"; +import { type NodeSqliteDatabaseOptions, openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import type { Storage } from "../src/types.ts"; + +/** The owner name the test storages use. */ +export const TEST_OWNER = "durable.test"; + +/** A `StoreStorage` on a store over the SQLite file at `path` (`:memory:` too) whose `close` closes the store too. */ +export async function openStoreStorage(path: string, options: NodeSqliteDatabaseOptions = {}): Promise { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path, options)), BACKGROUND_CONTEXT); + let storage: StoreStorage; + try { + storage = await StoreStorage.open(store, TEST_OWNER, BACKGROUND_CONTEXT); + } catch (error) { + await store.close(BACKGROUND_CONTEXT); + throw error; + } + let closing: Promise | undefined; + const close = (context: Parameters[0]): Promise => { + closing ??= storage.close(context).then(() => store.close(context)); + return closing; + }; + // StoreStorage keeps its state in private fields: its methods run on the storage itself, bound once. + const bound = new Map([["close", close]]); + return new Proxy(storage, { + get(target, property) { + let value = bound.get(property); + if (value !== undefined) return value; + value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + value = value.bind(target); + bound.set(property, value); + return value; + }, + // A test replacing a method (`storage.commit = …`) gets its own on the next read. + set(target, property, value) { + bound.delete(property); + return Reflect.set(target, property, value, target); + }, + }); +} diff --git a/packages/durable/test/store-storage.test.ts b/packages/durable/test/store-storage.test.ts new file mode 100644 index 000000000..94b7dee3a --- /dev/null +++ b/packages/durable/test/store-storage.test.ts @@ -0,0 +1,592 @@ +// StoreStorage (slice 6b §2): Durable's storage conformance suite on SQLite `:memory:` and across reopen, then what is +// its own: claims and close, owners sharing a store, the store's semantics for a refused commit, the allocator. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { Context } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT, withTelemetry } from "@earendil-works/chord/context"; +import { + applyBatch, + type ChordStore, + createHost, + defineFacet, + defineState, + importStore, + int, + local, + method, + obj, + openStore, + type Store, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { describe, expect, it } from "vitest"; +import { StorageRequestError } from "../src/errors.ts"; +import { idFromNumber } from "../src/ids.ts"; +import { type CompactionResult, createRegistry, Harness, LiveDoc, type Tx } from "../src/index.ts"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import type { ConversationId, DocumentId, EntryId, Id, Seq, Storage, StorageWrite, TaskId } from "../src/types.ts"; +import { ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +const context = BACKGROUND_CONTEXT; +const adopt = (): void => {}; + +/** A storage that closes and reopens its store file after every commit (Durable's reopen harness). */ +class ReopeningStorage implements Storage { + #current: Storage; + readonly #path: string; + #closed = false; + + constructor(current: Storage, path: string) { + this.#current = current; + this.#path = path; + } + + async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { + if (this.#closed) return this.#current.commit(writes, adopt, commitContext); + try { + return await this.#current.commit(writes, adopt, commitContext); + } finally { + await this.#current.close(context); + this.#current = await openStoreStorage(this.#path); + } + } + + mintId>(): Promise { + return this.#current.mintId(); + } + conversation: Storage["conversation"] = (id, c) => this.#current.conversation(id, c); + scanConversations: Storage["scanConversations"] = (q, l, cursor, c) => + this.#current.scanConversations(q, l, cursor, c); + entry(id: EntryId, c: Context): ReturnType; + entry(conversationId: ConversationId, id: EntryId, c: Context): ReturnType; + entry(first: EntryId | ConversationId, second: EntryId | Context, third?: Context) { + if (third === undefined) return this.#current.entry(idFromNumber(first), second as Context); + return this.#current.entry(idFromNumber(first), second as EntryId, third); + } + findLatestHeadMarker: Storage["findLatestHeadMarker"] = (id, at, c) => this.#current.findLatestHeadMarker(id, at, c); + scanEntries: Storage["scanEntries"] = (q, l, cursor, c) => this.#current.scanEntries(q, l, cursor, c); + task: Storage["task"] = (id, c) => this.#current.task(id, c); + scanTasks: Storage["scanTasks"] = (q, l, cursor, c) => this.#current.scanTasks(q, l, cursor, c); + submission: Storage["submission"] = (id, c) => this.#current.submission(id, c); + scanSubmissions: Storage["scanSubmissions"] = (q, l, cursor, c) => this.#current.scanSubmissions(q, l, cursor, c); + submissionByRequest: Storage["submissionByRequest"] = (id, request, c) => + this.#current.submissionByRequest(id, request, c); + findDocument: Storage["findDocument"] = (address, at, c) => this.#current.findDocument(address, at, c); + document: Storage["document"] = (id, at, c) => this.#current.document(id, at, c); + scanDocuments: Storage["scanDocuments"] = (q, l, cursor, c) => this.#current.scanDocuments(q, l, cursor, c); + + async close(closeContext: Context): Promise { + if (this.#closed) return; + this.#closed = true; + await this.#current.close(closeContext); + } +} + +registerStorageConformance({ describe, expect, it }, "StoreStorage", async (use) => { + const storage = await openStoreStorage(":memory:"); + try { + await use(storage); + } finally { + await storage.close(context); + } +}); + +registerStorageConformance({ describe, expect, it }, "StoreStorage across reopen", async (use) => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-conformance-")); + const path = join(directory, "store.sqlite"); + const storage = new ReopeningStorage(await openStoreStorage(path), path); + try { + await use(storage); + } finally { + await storage.close(context); + await rm(directory, { recursive: true, force: true }); + } +}); + +/** A store on `:memory:`. */ +async function memoryStore() { + return openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +} + +describe("claims and close", () => { + it("refuse a second StoreStorage on one owner until the first closed; two owners share a store", async () => { + const store = await memoryStore(); + const first = await StoreStorage.open(store, "harness", context); + await expect(StoreStorage.open(store, "harness", context)).rejects.toMatchObject({ + code: "invalid", + reason: "args", + }); + const other = await StoreStorage.open(store, "other", context); + await first.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + expect(await other.conversation(ROOT_CONVERSATION_ID, context)).toBeUndefined(); + await first.close(context); + await expect(first.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("StoreStorage is closed"); + const reopened = await StoreStorage.open(store, "harness", context); + expect(await reopened.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); + + it("waits at close for its calls underway, then releases its claims", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const commit = storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const closing = storage.close(context); + await commit; + await closing; + await StoreStorage.open(store, "harness", context); + await store.close(context); + }); +}); + +describe("the store's semantics (D4)", () => { + it("a refused Durable commit rejects with Durable's message and keeps the store usable", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const root = { type: "conversation", value: { id: ROOT_CONVERSATION_ID } } as const; + await storage.commit([root], adopt, context); + await expect(storage.commit([root], adopt, context)).rejects.toThrow("ID 1 already belongs to conversation"); + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + await storage.commit([{ type: "entry", value: entry }], adopt, context); + expect((await storage.entry(entry.id, context))?.entry).toEqual(entry); + await store.close(context); + }); + + it("raises the allocator above every id it holds when it opens (a store built by import)", async () => { + const store = await memoryStore(); + const storage = await StoreStorage.open(store, "harness", context); + const entry = { id: idFromNumber(500), conversationId: ROOT_CONVERSATION_ID, kind: "m" }; + await storage.commit( + [ + { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, + { type: "entry", value: entry }, + ], + adopt, + context, + ); + expect(await storage.mintId()).toBe(501); + await store.close(context); + }); +}); + +const MemoDoc = defineState("memo", { + schema: obj({ m: int() }), + version: 1, + initial: () => ({ m: 1 }), + persist: true, + checkpointWhen: () => true, +}); + +const Memo = local("app.memo", { version: 1, params: {}, members: { memo: obj({ m: int() }), set: method() } }); + +/** A store on `:memory:` with an outbox from its first commit, and Durable's storage in it. */ +async function recorded() { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, { + outbox: true, + })) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + return { store, storage }; +} + +/** Rewindable history, a retired latest document, a create-plus-retire and a change-plus-retire. */ +async function history(storage: Storage): Promise<{ readonly rewindable: DocumentId; readonly seqs: Seq[] }> { + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const rewindable = await storage.mintId(); + const latest = await storage.mintId(); + const empty = await storage.mintId(); + const seqs: Seq[] = []; + const base = (n: number) => ({ kind: "base", version: 1, value: { n } }) as const; + seqs.push( + await storage.commit( + [ + { + type: "document.create", + record: { id: rewindable, kind: "r", scope, history: "rewindable", fork: "asOf" }, + content: base(0), + }, + { + type: "document.create", + record: { id: latest, kind: "l", scope, history: "latest", fork: "current" }, + content: base(0), + }, + ], + adopt, + context, + ), + ); + seqs.push( + await storage.commit( + [{ type: "document.change", id: rewindable, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + adopt, + context, + ), + ); + seqs.push( + await storage.commit( + [ + { type: "document.change", id: latest, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }, + { type: "document.retire", id: latest }, + { type: "document.create", record: { id: empty, kind: "e", scope: { kind: "session" } }, content: base(3) }, + { type: "document.retire", id: empty }, + ], + adopt, + context, + ), + ); + return { rewindable, seqs }; +} + +describe("archives and batches of Durable data", () => { + it("export and import keep rewindable history, retired records and Chord's own data: reads at a point agree", async () => { + const { store, storage } = await recorded(); + const { rewindable, seqs } = await history(storage); + // Chord's own data in the same store: an object's persisted document. + const host = createHost({ store, onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Memo, (_, self) => { + const memo = self.state(MemoDoc); + return { + memo, + set: async (ctx) => { + await memo.change((draft) => { + draft.m = 2; + }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + await (await host.get(Memo, {}, context)).set(context); + await host.close(context); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "document.create").map(({ record }) => record.owner)).toContain( + "app.memo", + ); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + const imported = await StoreStorage.open(copy, "harness", context); + for (const at of [seqs[0]!, seqs[1]!, "current" as const]) { + expect(await imported.document(rewindable, at, context)).toEqual( + await storage.document(rewindable, at, context), + ); + } + const scope = { kind: "conversation", conversationId: ROOT_CONVERSATION_ID } as const; + const at = seqs[1]!; + expect(await imported.scanDocuments({ scope, at }, 10, undefined, context)).toEqual( + await storage.scanDocuments({ scope, at }, 10, undefined, context), + ); + const again = []; + for await (const part of copy.export(context)) again.push(part); + expect(again).toEqual(parts); + expect(await imported.mintId()).toBe(await storage.mintId()); + await copy.close(context); + await store.close(context); + }); + + it("an outbox of Durable commits replayed into an empty store gives the same archive (create-plus-retire included)", async () => { + const { store, storage } = await recorded(); + await history(storage); + const commits = await (store as ChordStore).outbox!.read(0, 1_000, context); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, commits, context); + const exported = async (from: Store) => { + const parts = []; + for await (const part of from.export(context)) parts.push(part); + return parts; + }; + expect(await exported(replica)).toEqual(await exported(store)); + await replica.close(context); + await store.close(context); + }); +}); + +describe("head markers (the entry_heads table)", () => { + it("a head entry is found after reopen, in an imported archive and in a replayed outbox", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-store-heads-")); + const path = join(directory, "store.sqlite"); + const open = async () => + (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: true })) as ChordStore; + let store = await open(); + let storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const entry = async (head?: EntryId): Promise => { + const id = await storage.mintId(); + const value = { id, conversationId: ROOT_CONVERSATION_ID, kind: "k", data: {}, ...(head ? { head } : {}) }; + await storage.commit([{ type: "entry", value }], adopt, context); + return id; + }; + const first = await entry(); + const marker = await entry(first); + await entry(); + /** The newest head marker, and the one at or before the first entry (none). */ + const heads = async (of: Storage) => [ + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id, + (await of.findLatestHeadMarker(ROOT_CONVERSATION_ID, first, context))?.id, + ]; + expect(await heads(storage)).toEqual([marker, undefined]); + await storage.close(context); + await store.close(context); + store = await open(); + storage = await StoreStorage.open(store, "harness", context); + expect(await heads(storage)).toEqual([marker, undefined]); + const parts = []; + for await (const part of store.export(context)) parts.push(part); + expect(parts.filter((part) => part.kind === "put" && part.table === "harness:entry_heads")).toEqual([ + expect.objectContaining({ row: { id: marker, conversation_id: ROOT_CONVERSATION_ID } }), + ]); + const copy = await importStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), parts, context); + expect(await heads(await StoreStorage.open(copy, "harness", context))).toEqual([marker, undefined]); + const replica = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + await applyBatch(replica, await store.outbox!.read(0, 1_000, context), context); + expect(await heads(await StoreStorage.open(replica, "harness", context))).toEqual([marker, undefined]); + for (const each of [copy, replica, store]) await each.close(context); + await rm(directory, { recursive: true, force: true }); + }); +}); + +describe("reads (slice 6b §1.4, §2.4)", () => { + it("a request error inside a multi-query read rejects that read only; the store goes on", async () => { + const { store, storage } = await recorded(); + await expect( + storage.scanEntries({ conversationId: idFromNumber(999) }, 10, undefined, context), + ).rejects.toBeInstanceOf(StorageRequestError); + await expect(storage.document(idFromNumber(999), "current", context)).resolves.toBeUndefined(); + expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); + await store.close(context); + }); +}); + +describe("commits (slice 6b §1.3, §2.5)", () => { + it("adopt in the attached host's batch, and count four statements for an entry append and for a delta", async () => { + const telemetry = new InMemoryTelemetryContext(); + const span = await telemetry.startSpan({ name: "test" }, (each) => each); + const metered = withTelemetry(span, context); + const counts = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.store.statements") + .reduce((total, { value }) => total + value, 0); + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + const host = createHost({ store, onError: () => {} }); + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + let running: boolean | undefined; + const entry = { + id: idFromNumber(await storage.mintId()), + conversationId: ROOT_CONVERSATION_ID, + kind: "m", + }; + const before = counts(); + await storage.commit( + [{ type: "entry", value: entry }], + () => { + running = store.batch!.running; + }, + metered, + ); + const appended = counts() - before; + expect(running).toBe(true); + const id = await storage.mintId(); + const record = { id, kind: "s", scope: { kind: "session" } } as const; + await storage.commit( + [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 0 } } }], + adopt, + context, + ); + const changed = counts(); + await storage.commit( + [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], + adopt, + metered, + ); + expect([appended, counts() - changed]).toEqual([4, 4]); + await host.close(context); + await store.close(context); + }); +}); + +describe("Durable's views as Chord sources on the store (slice 6b §3)", () => { + const Chat = local("app.chat", { + version: 1, + params: {}, + members: { entries: int(), note: method(obj({ kind: str() })) }, + }); + + it("a derive over the conversation view and the task graph: one frame per Durable commit, both views in one batch", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + let computes = 0; + const facet = defineFacet((h) => + h.serve(Chat, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Chat.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const view = self.follow((observer, ctx) => + harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + const graph = self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + const entries = self.derive([view, graph], ([v]) => { + computes++; + return v.entries.length; + }); + return { + entries, + note: async ({ kind }, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const chat = await host.get(Chat, {}, context); + const calls: unknown[] = []; + const watch = await chat.entries.watch(context); + watch.start(async (call) => void calls.push(call.kind === "value" ? call.value : call.kind)); + const before = computes; + await chat.note({ kind: "test.note" }, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(chat.entries.value).toBeGreaterThan(0); + expect(calls).toEqual([chat.entries.value]); + expect(computes - before).toBe(1); + await host.close(context); + await store.close(context); + }); + + it("a deferred Durable commit, then an immediate one, reach the followed document in order", async () => { + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const Live = local("app.live", { + version: 1, + params: {}, + members: { n: int(), bump: method() }, + }); + /** A commit setting the root's `LiveDoc` failure note to `value`. */ + const set = (value: number) => async (tx: Tx) => { + (await tx.doc(LiveDoc, ROOT_CONVERSATION_ID)).compactionFailure = { + taskId: idFromNumber>(1), + trigger: "manual", + error: String(value), + }; + }; + const facet = defineFacet((h) => + h.serve(Live, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Live.name, open), + { models: createModels(), registry: createRegistry() }, + open, + ); + await (await opened.root(open)).commit(set(0), open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + const live = self.follow((observer, ctx) => + harness.value.observeDoc(LiveDoc, ROOT_CONVERSATION_ID, observer, ctx), + ); + return { + n: self.derive([live], ([value]) => Number(value.compactionFailure?.error ?? -1)), + bump: async (ctx) => { + const root = (await harness.value.conversation(ROOT_CONVERSATION_ID, ctx))!; + await root.commit(set(1), ctx, { persistence: { kind: "deferred", withinMs: 10_000 } }); + await root.commit(set(2), ctx); + return null; + }, + }; + }), + ); + await host.load([{ name: "app", facet }], context); + const live = await host.get(Live, {}, context); + const seen: unknown[] = []; + const watch = await live.n.watch(context); + watch.start(async (call) => void seen.push(call.kind === "value" ? call.value : call.kind)); + await live.bump(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(live.n.value).toBe(2); + expect(seen.at(-1)).toBe(2); + await host.close(context); + await store.close(context); + }); +}); + +describe("review regressions", () => { + it('a head-marker entry whose kind is "refused" is returned, not taken for a request error', async () => { + const storage = await openStoreStorage(":memory:"); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const first = await storage.mintId(); + const marker = await storage.mintId(); + await storage.commit( + [ + { type: "entry", value: { id: first, conversationId: ROOT_CONVERSATION_ID, kind: "message" } }, + { + type: "entry", + value: { id: marker, conversationId: ROOT_CONVERSATION_ID, kind: "refused", head: first }, + }, + ], + adopt, + context, + ); + expect((await storage.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id).toBe(marker); + const page = await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, context); + expect(page.items.map(({ kind }) => kind)).toEqual(["refused", "message"]); + await storage.close(context); + }); + + it("an empty commit takes no seq", async () => { + const { store, storage } = await recorded(); + const before = store.lastSeq; + const adopted: Seq[] = []; + expect(await storage.commit([], (seq) => adopted.push(seq), context)).toBe(before); + expect(store.lastSeq).toBe(before); + expect(adopted).toEqual([before]); + await store.close(context); + }); +}); + +describe("GC", () => { + it("a closed StoreStorage is collected while its store stays open", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const store = await memoryStore(); + const opened = async (): Promise> => { + const storage = await StoreStorage.open(store, "harness", context); + await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await storage.close(context); + return new WeakRef(storage); + }; + const ref = await opened(); + for (let round = 0; round < 10 && ref.deref() !== undefined; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(ref.deref()).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/durable/vitest.store.config.ts b/packages/durable/vitest.store.config.ts new file mode 100644 index 000000000..b98674172 --- /dev/null +++ b/packages/durable/vitest.store.config.ts @@ -0,0 +1,47 @@ +// The harness test files that take a SQLite storage, run on Durable's storage on Chord's store (`StoreStorage`, slice +// 6b): `../src/storage/sqlite/node.ts`'s opener is replaced by `test/store-storage-node.ts`; the files are unchanged. +// Four tests edit or read `SqliteStorage`'s own tables with raw SQL (`tasks`, `durable_metadata`) and are left out. + +import { fileURLToPath } from "node:url"; +import { defineConfig, mergeConfig } from "vitest/config"; +import base from "./vitest.config.ts"; + +const storeOpener = fileURLToPath(new URL("./test/store-storage-node.ts", import.meta.url)); + +export default mergeConfig( + base, + defineConfig({ + test: { + testNamePattern: + /^(?!.*(resumes a retry backoff after reopen|sleeps again for a retry stored as a running phase|polls a checkpoint stored without a thinking level|refuses to create the root of an empty storage))/, + include: [ + "test/harness-after-run.test.ts", + "test/harness-blocked.test.ts", + "test/harness-compaction.test.ts", + "test/harness-continue.test.ts", + "test/harness-conversations.test.ts", + "test/harness-generation-recovery.test.ts", + "test/harness-inbox.test.ts", + "test/harness-input-entries.test.ts", + "test/harness-lifecycle.test.ts", + "test/harness-nested-tools-restart.test.ts", + "test/harness-nested-tools.test.ts", + "test/harness-ownership.test.ts", + "test/harness-read-only.test.ts", + "test/harness-restart-failure.test.ts", + "test/harness-scheduler-indexes.test.ts", + "test/harness-scheduler-retention.test.ts", + "test/harness-sleep.test.ts", + "test/harness-structured.test.ts", + "test/harness-submissions.test.ts", + "test/harness-task-graph.test.ts", + "test/harness-tasks-recovery.test.ts", + "test/harness-tool-hooks.test.ts", + "test/harness-tools-recovery.test.ts", + "test/harness-turn-limit.test.ts", + "test/harness-view.test.ts", + ], + }, + resolve: { alias: [{ find: /^\.\.\/src\/storage\/sqlite\/node\.ts$/, replacement: storeOpener }] }, + }), +); diff --git a/tsconfig.json b/tsconfig.json index 8a90126b1..eea4c716b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,7 @@ "@earendil-works/chord": ["./packages/chord/src/index.ts"], "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], + "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"],