diff --git a/README.md b/README.md index 9c88e5bf2..93184c10f 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ This monorepo contains the Pi CLI and its supporting libraries. | Package | Description | |---------|-------------| -| **[@earendil-works/chord](packages/chord)** | Standalone application-composition runtime for services, replicated state, RPC, and plugins | +| **[@earendil-works/chord](packages/chord)** | Typed host runtime for hot-reloadable plugins, contracts, replicated state, and RPC over JSON links | | **[@earendil-works/pi-telemetry](packages/telemetry)** | Vendor-neutral telemetry contracts, reference adapter, conformance tests, and typed schemas | | **[@earendil-works/pi-ai](packages/ai)** | Unified multi-provider LLM API (OpenAI, Anthropic, Google, etc.) | | **[@earendil-works/pi-durable](packages/durable)** | Durable conversation, task, and document runtime | diff --git a/package-lock.json b/package-lock.json index 9c236ad71..4216448ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -716,6 +716,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -732,6 +733,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -748,6 +750,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -764,6 +767,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -780,6 +784,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -796,6 +801,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -812,6 +818,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -828,6 +835,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -844,6 +852,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -860,6 +869,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -876,6 +886,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -892,6 +903,7 @@ "cpu": [ "loong64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -908,6 +920,7 @@ "cpu": [ "mips64el" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -924,6 +937,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -940,6 +954,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -956,6 +971,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -972,6 +988,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -988,6 +1005,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1004,6 +1022,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1020,6 +1039,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1036,6 +1056,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1052,6 +1073,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1068,6 +1090,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1084,6 +1107,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1100,6 +1124,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1116,6 +1141,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2665,6 +2691,7 @@ "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -5107,7 +5134,7 @@ "version": "1.1.0", "license": "MIT", "dependencies": { - "esbuild": "0.28.2" + "typebox": "1.3.27" }, "devDependencies": { "shx": "0.3.4", diff --git a/packages/chord/PLANNING.md b/packages/chord/PLANNING.md deleted file mode 100644 index 03e34ba9c..000000000 --- a/packages/chord/PLANNING.md +++ /dev/null @@ -1,859 +0,0 @@ -# Chord implementation plan - -> **Status:** Active implementation plan. Context, strict JSON, replicated state, service publication/consumption, the facet host/loader, and initial Node facet bundling and generation loading now live in Chord. Symmetric RPC and structural generation replacement remain planned. This is not a stable public API contract yet. - -## 1. Goal - -Chord will be the application-neutral foundation for: - -1. loading, composing, unloading, reloading, and bundling plugins; -2. declaring and consuming local or remote services; -3. transporting service calls and subscriptions over symmetric RPC plumbing; and -4. replicating authoritative latest-value state to local and remote consumers. - -The current Pi experiments prove many required behaviors, but Chord will be implemented from scratch. Existing source may be used as test and design evidence, not copied into this package. Compatibility with experimental APIs or wire messages is not a requirement. - -## 2. Dependency boundary - -The dependency direction is strict: - -```text -@earendil-works/chord - ↑ -Pi agent, protocol, server, coding agent, TUI, and future applications -``` - -Chord must: - -- have no dependency on another Pi workspace package; -- contain no imports from `@earendil-works/pi-*` or relative paths outside `packages/chord`; -- use application-neutral vocabulary in source, errors, tests, and examples; -- own any generic runtime types required by its public API, including strict JSON values and invocation cancellation context; -- keep Node-specific loading and bundling separate from the platform-neutral runtime; and -- be buildable, testable, packable, and usable without resolving another Pi package. - -A generic third-party dependency is not prohibited, but each dependency must be justified. The runtime should initially prefer standard JavaScript APIs. The bundler may use one pinned implementation behind a Chord-owned adapter. - -The following terms must not become Chord concepts: Session, Harness, AgentLane, server, client, attachment, TUI, model, tool, hook, provider credential, or workspace. Those belong to consumers. - -## 3. Architectural model - -### 3.1 Working vocabulary - -The names below are provisional, but the distinctions are required. - -- **Plugin**: one independently activated composition unit with a stable ID and synchronous setup function. -- **Plugin module**: a JavaScript module exporting one or more plugins for one application-selected entry. -- **Loaded generation**: plugins plus the resources owned by loading their module generation. -- **Host**: one assembled plugin and service graph. -- **Service token**: a stable runtime service ID plus type information and locality policy. -- **Provider**: the owner of one singleton service or one keyed service collection. -- **Connection**: a transport-neutral source of services outside the current host. -- **Peer**: one endpoint of a symmetric RPC channel. Either peer may provide and consume services. -- **Replicated state**: initialized immutable source revisions with read-only local or remote replicas. - -A product feature may ship multiple plugin-module entries for different application environments. Chord does not group those entries into a cross-process runtime object and does not interpret their entry names. - -### 3.2 Layering - -The implementation should be divided into these layers: - -```text -plugin loader and bundler - ↓ -plugin host, lifecycle, and dependency graph - ↓ -service tokens, providers, facades, and keyed instances - ↓ -replicated state and service subscriptions - ↓ -symmetric RPC peer and transport adapter - ↓ -strict JSON, invocation context, cancellation, and errors -``` - -The local service path must not require RPC serialization. The remote path must use the same service semantics through a strict wire boundary. - -## 4. Invocation context and strict JSON - -Current experiments depend on Pi's Harness `Context`, which Chord cannot import. Chord therefore needs a small neutral invocation context. - -The initial context should provide: - -- an optional `AbortSignal` for cancellation of one invocation or observation; -- immutable typed local values through Chord-owned context keys; -- a background root; -- child derivation, cancellation, and cancellation-aware waiting helpers; and -- no built-in telemetry, identity, authentication, or application values. - -Applications may define their own context keys. A Pi adapter can carry telemetry and authenticated identity through such keys without Chord knowing their types. - -A context object never crosses RPC as a business value. The calling peer sends cancellation control and, if configured, an opaque strict-JSON metadata carrier. The receiving adapter constructs a fresh local context. The adapter, not remote business arguments, installs authenticated local identity. - -Chord owns the static `JsonValue` contract and provides `JsonRepresentation` plus `isJsonValue()` for adapter boundaries. The service runtime deliberately performs no automatic recursive validation; concrete serializers remain responsible for rejecting unsupported values. Remote arguments, results, errors, snapshots, updates, catalogues, and RPC envelopes are expected to be finite strict JSON: - -- finite numbers only; -- no `undefined`, sparse arrays, symbols, prototypes, cycles, classes, functions, `Map`, or `Set`; and -- `null`, rather than `undefined`, for business-level absence. - -Application schema validation remains the application's responsibility. Chord validates structural control envelopes, but runtime enforcement of the strict-JSON boundary is currently deferred to serializers. - -## 5. Plugins and lifecycle - -### 5.1 Plugin shape - -The intended author model is equivalent to: - -```ts -interface Plugin { - readonly id: string; - setup(environment: PluginEnvironment): void; -} -``` - -Setup is synchronous declaration. It may: - -- provide a singleton service; -- declare ownership of a keyed service; -- acquire a singleton service handle; -- declare an observation of keyed service instances; -- create replicated state; -- register an activation callback; -- register owned resource cleanup; and -- register final deactivation work. - -Setup must not: - -- invoke a service or read replicated state through an acquired facade; -- perform asynchronous work; -- introduce a new service dependency later from an event handler or activation callback; or -- mutate the active host generation. - -The host records setup calls in a private generation ledger. Plugin authors do not maintain a parallel `requires`/`provides` manifest. - -### 5.2 Assembly - -After every plugin has completed setup, the host must: - -1. collect local provisions and catalogues from configured remote service sources; -2. resolve every hard requirement to exactly one local or connected provision; -3. reject missing providers, duplicate providers, ambiguous source offers, and singleton/keyed mismatches; -4. reject duplicate plugin IDs; -5. derive provider-to-consumer lifecycle edges; -6. reject dependency cycles; -7. construct local and remote service bindings while handles are still inaccessible; -8. hydrate required connected services; and -9. activate providers before consumers. - -A plugin may provide and consume the same token without creating a self-cycle. Optional dependencies are not part of the first contract; they require a distinct acquisition API later. - -### 5.3 Resource ownership - -Every plugin generation owns: - -- activation callbacks; -- explicit cleanup functions; -- keyed instances added through its provider handle; -- keyed observations and their tasks; and -- service provisions. - -Disposal is idempotent. Consumers deactivate before providers, and each plugin's resources dispose in reverse registration order. Cleanup continues after individual failures and reports one error or an `AggregateError` after all cleanup attempts finish. - -Keyed observation handlers receive a fresh cancellable context. Closing or replacing the instance aborts only that handler task. A handler failure is reported through host policy unless its context was cancelled as normal cleanup. - -### 5.4 Loaded module generations - -Module loading and plugin activation are separate ownership domains: - -```ts -interface LoadedPlugins { - readonly plugins: readonly Plugin[]; - dispose(): Promise; -} - -interface PluginLoader { - load(): Promise; -} -``` - -Required loaders: - -- a static loader for built-ins and tests; -- an ordered combined loader with reverse disposal and startup-failure cleanup; and -- a bundle/module loader for one selected manifest entry. - -The loader owns one source generation. The host owns active plugin lifecycles. A coordinator must deactivate a retired generation before disposing its `LoadedPlugins`. - -Node's default ESM loader retains every imported module generation in its process-wide cache. Chord therefore bundles Node facets as CommonJS and compiles each generation directly with `node:vm`, without inserting plugin code into either Node module cache. Chord's unload contract remains deactivation, removal of service reachability, cleanup, and release of loader-owned references/resources. Once no plugin-created timer, listener, callback, or other escaped reference remains, V8 can garbage-collect the compiled generation; collection timing is not deterministic. - -## 6. Loading, unloading, and reload - -### 6.1 Host updates - -Host updates are serialized and have two forms. - -#### Shape-preserving replacement - -A targeted replacement is shape-preserving when each replaced plugin keeps the same: - -- plugin ID; -- required service IDs and modes; -- provided service IDs and modes; and -- remotely exposable singleton member names and kinds. - -The sequence is: - -```text -load replacement module generation -→ run replacement setup -→ validate replacement shape and remote implementations -→ activate replacements in dependency order while the old providers remain routed -→ replace each local or remote singleton directly without withdrawing it -→ deactivate replaced plugins in reverse dependency order -→ dispose the retired loaded generation -``` - -Properties: - -- existing local and remote singleton facades keep object identity; -- captured local and remote methods dispatch to the old provider before cutover and the replacement after cutover; -- replicated-state facades install the replacement snapshot without becoming unhydrated; -- every singleton switches directly from its old target to its replacement, but a multi-service reload is not graph-transactional; -- keyed instances from the old plugin close; instances staged by the replacement use fresh generations; -- setup, shape-validation, or replacement-activation failure leaves the active generation unchanged; -- named host resources acquired during activation support overlapping staged replacements, so candidate cleanup restores the old registration and retired cleanup cannot remove the replacement; and -- any failure after replacement begins terminates the host rather than attempting to preserve a partially transitioned graph. - -There is no rollback after cutover. Terminal cleanup revokes every facet handle before best-effort disposal; only pre-cutover candidate cleanup and ordinary host disposal guarantee dependency-ordered cleanup. Committed application effects are outside the reload transaction. - -#### Structural replacement - -Adding or removing plugins, changing service shape, or changing connection selection is structural. The first implementation should replace the complete host generation rather than attempt a partial affected-subgraph update: - -```text -load and synchronously set up the complete desired generation -→ resolve and validate its graph without activation effects -→ begin cutover -→ withdraw old providers and deactivate the old graph -→ install and activate the new graph -→ dispose the old loaded generation -``` - -Rules: - -- validation failure before cutover leaves the old graph active; -- after cutover starts, failure disposes the complete host because the retired graph cannot be restored; -- removing a provider while retaining a hard consumer is rejected during candidate validation; -- service facades owned outside plugin lifecycles remain stable when service ID and mode survive the replacement; -- removed services become permanently disconnected for the retired generation; and -- a future optimization may retain unaffected plugins, but it is not required for the initial structural update. - -This full-generation path supplies actual plugin load and unload semantics without first implementing a complex partial-graph transaction. - -### 6.2 Calls during unload - -Replacement never leaves a singleton facade without a target: an invocation resolves either the old implementation or its replacement. Work already running in a retired facet is not drained; later use of that facet's revoked handles may fail as stale work. A future killable-isolate host terminates such work directly. Transport disconnect still cancels only connection-owned invocations; it does not imply application-level cancellation or rollback. - -### 6.3 Update failure reporting - -An update result must distinguish: - -- load failure; -- setup or graph validation failure before cutover; -- old-generation deactivation failure; -- replacement activation failure; -- service rebinding failure; and -- retired loader disposal failure. - -Multiple failures are aggregated without hiding the first transition failure. Concurrent update, unload, and host-dispose requests are serialized or rejected with a stable lifecycle error. - -## 7. Services - -### 7.1 Tokens and modes - -A service token has: - -- a non-empty stable string ID; -- a TypeScript contract type; -- a locality policy: remotely exposable by default or explicitly process-local; and -- no provider instance of its own. - -Chord reserves a prefix for control-plane IDs. Duplicate IDs in one catalogue are invalid. - -Two modes are required: - -- **singleton**: one provider, many consumers; and -- **keyed**: one collection owner, dynamic instances, many observers. - -One token has one mode in one host graph. Mixing singleton and keyed use is an assembly or protocol error. - -### 7.2 Local services - -Process-local services: - -- may expose arbitrary objects, synchronous methods, classes, functions, native handles, or non-JSON values; -- are never included in a remote catalogue; -- use the same graph ordering, stable handles, keyed generations, and lifecycle behavior as remote services; and -- are trusted composition, not a security boundary. - -A local singleton consumer receives a stable lazy facade, not the provider object. This removes setup-order dependence and lets provider replacement update captured methods. - -Remotely exposable services use the provider/binding path even when provider and consumer share a host. An internal loopback binding keeps replacement, replicated-state, and keyed-generation semantics independent of placement. Only explicitly process-local services bypass that path. - -### 7.3 Remote service contracts - -A remotely exposable implementation may contain only own data properties classified as: - -- asynchronous methods whose business arguments and result are strict JSON, with one Chord invocation context in the declared position; or -- branded replicated-state values. - -The provider derives a runtime member table from the implementation. Plugin authors do not maintain a second method/state descriptor. Unsupported accessors, fields, or member kinds are rejected before publication. - -Type-level checks should reject obviously invalid remote contracts. Runtime checks remain mandatory because types do not authenticate peers or survive JavaScript consumption. - -Remote methods: - -- return promises; -- return strict JSON or `void`; -- receive a fresh local invocation context; -- map caller cancellation to exactly one request; -- do not queue while disconnected; and -- expose stable error codes and sanitized messages across the wire. - -Arbitrary returned object references, callbacks, function serialization, and general object-graph remoting are outside the initial scope. - -### 7.4 Stable singleton facades - -`use(token)` returns a facet-owned capability view over a source-independent host service slot. Repeated acquisition within one facet returns the same view; different facet lifecycles receive different views. Member slots are created lazily on property access and validated against provider metadata when bound. - -Required behavior: - -- inaccessible during setup and after the owning facet lifecycle ends; -- directly bound after local graph assembly or remote hydration; -- stable across provider withdrawal and replacement; -- method invocation fails while disconnected; -- state reads return `undefined` while unhydrated; -- a remotely exposable replacement preserves the complete member name/kind table; and -- a provider omitting an already-accessed member is a binding error. - -### 7.5 Keyed services - -`provideMany(token)` declares one keyed owner during setup and returns a generation-owned `ServiceSpawner`. While active, `spawn(key, implementation)`: - -- requires a non-empty key unique among live instances; -- creates a host-owned monotonically increasing generation for that key; -- publishes methods and initial state atomically; -- returns an idempotent close function; and -- is automatically closed during plugin disposal. - -An instance address is `(service ID, key, generation)`. Reusing a closed key creates a new generation. Stale facades cannot call the replacement. - -`observe(token, handler)`: - -- reconciles a complete initial instance directory; -- starts one task per instance only after all initial state members hydrate; -- preserves ordered additions, replacements, and removals; -- aborts the instance task on close, replacement, disconnect, or observation disposal; -- gives each observation an observer-lifetime service view that becomes inaccessible when its task is aborted; and -- never represents service facades inside replicated JSON state. - -## 8. Replicated state - -Replicated state is authoritative one-writer latest-value replication. - -The source API has an initialized immutable value, `change(context, callback)`, `replace(context, value)`, and subscriptions. A remote or disconnected replica has `value === undefined` until hydration. - -Required semantics: - -1. The source is always initialized. -2. Source and replica values are strict JSON for remotely exposable state. -3. Subscribing to initialized local or hydrated replica state immediately delivers the current value with a fresh delivery context. -4. Subscribing to cold state registers without immediate delivery. -5. Subscription establishment installs update capture before taking the snapshot. -6. Updates racing the snapshot are buffered and delivered after the snapshot with no gap. -7. The source API exposes atomic copy-on-write transactions. A successful transaction publishes one decoded operation batch; a callback failure discards all tentative copies. Connection adapters encode each batch independently per client/state stream, and replicas apply batches only in sequence order. -8. A sequence gap clears readiness and triggers complete resubscription or reports a terminal binding error; stale state must not continue as current silently. -9. Disconnect, provider withdrawal, route change, and replacement clear replica readiness. -10. Reconnect or replacement installs a complete fresh snapshot in the existing state facade before later updates. -11. Listener exceptions are isolated and reported through host policy. -12. Values are immutable data. Chord returns raw immutable values for reads and listener delivery. Transaction drafts are revocable and assigned containers are copied by value; committed revisions structurally share unchanged subtrees. - -State identity is structural: - -```text -provider binding + service ID + optional keyed address + member name -``` - -There is no separate state ID. - -Explicit non-goals: - -- durability or reconstruction after process restart; -- event history; -- CRDT merging or multiple writers; -- offline mutation replay; -- high-frequency stream transport. - -Chord exposes an intent-preserving JSON delta primitive and uses its operation batches internally for remote replicated state. Initial hydration and reconnection carry a complete root replacement. Producers mutate a transaction-scoped copy-on-write draft; Chord derives compact operations from the previous and next immutable revisions and uses a complete replacement when that is smaller. Replicated-state sources do not select reducers or interact with path encoders. Every client/state pairing owns an independent encoder, and sequence handling rejects gaps before a later operation can be applied. - -## 9. Symmetric RPC plumbing - -### 9.1 No client/server mode - -Chord must not expose `Client`, `Server`, `SessionConnection`, or similar topology classes. It exposes a symmetric peer over an application-supplied duplex channel. Either endpoint may register handlers, provide services, call methods, or subscribe to state. - -Application adapters own: - -- sockets, pipes, WebSockets, workers, or loopback delivery; -- framing and reconnect policy; -- authentication and authorization; -- routing and target selection; -- process ownership; -- attachment or selection state; and -- application protocol envelopes surrounding a Chord message. - -Chord sees one connected peer and already-decoded strict-JSON messages. - -### 9.2 RPC layer - -The generic RPC peer should own: - -- request ID allocation scoped to one peer connection; -- request/response correlation; -- handler registration; -- cancellation frames and one request-local `AbortController`; -- disconnect rejection and cleanup; -- stable serializable error envelopes; -- ordered notification delivery; -- malformed-message rejection; and -- optional context metadata hooks. - -The channel contract should be small: send one strict-JSON message, receive messages in order, observe close, and close. `send()` should be awaitable so adapters can provide backpressure. - -A peer disconnect: - -- rejects outbound requests; -- aborts inbound request contexts; -- closes service subscriptions and keyed observation tasks; and -- does not cancel application-owned jobs or perform durable business mutations. - -### 9.3 Service protocol over RPC - -The service layer adds transport-neutral operations for: - -- catalogue discovery; -- singleton and keyed subscription open/close; -- complete subscription snapshots; -- method invocation; -- invocation cancellation; -- replicated-state updates; -- singleton unavailable/replaced events; and -- keyed instance spawned/closed events. - -The wire protocol contains no server ID, session ID, attachment ID, route, user identity, or host kind. A Pi router may wrap or forward Chord envelopes using its own control fields without parsing service business payloads. - -Provider catalogue and subscription state must come from actual service provisions, not a handwritten application inventory. - -### 9.4 Wire errors and validation - -Chord needs stable generic error codes for: - -- service not allowed or absent; -- mode mismatch; -- member absent or used with the wrong kind; -- keyed instance absent or stale; -- invalid strict-JSON value; -- cancellation; -- malformed RPC message; -- peer disconnected; and -- internal provider failure. - -Unexpected provider exceptions become a sanitized internal error by default. Stack traces and arbitrary exception fields do not cross the wire. Applications may register or map additional stable error codes through an adapter, but Chord does not own application error taxonomies. - -### 9.5 Protocol evolution - -The first implementation must version Chord's RPC/service envelope independently of any application protocol. Version negotiation can be a peer handshake or an adapter-guaranteed constructor parameter, but incompatible peers must fail before service calls are admitted. - -Member and DTO compatibility across plugin-generation skew is an application responsibility. Chord guarantees only its generic envelope semantics. - -## 10. Bundling - -### 10.1 Purpose - -The bundler turns one or more application-declared ESM or TypeScript plugin entries into independently loadable Node CommonJS artifacts. It is not a package manager or plugin registry. - -Each entry is built independently. Chord does not assume names such as `server`, `session`, `tui`, or `web`; entry names are opaque application data. - -### 10.2 Initial input and output - -The initial bundler should accept: - -- plugin identity and optional version metadata; -- a mapping of opaque entry names to TypeScript or JavaScript source files; -- an output directory; -- an application-supplied external-module allowlist; -- source-map and minification options; and -- optional define/platform settings needed for the Node CommonJS build. - -The package-level API additionally accepts a plugin package directory, derives identity and version from `package.json`, applies application-supplied conventional entry paths when those files exist, and lets `chord.facets` override or disable conventions. Package discovery does not install dependencies or run lifecycle scripts. - -It should emit: - -- one content-addressed CommonJS file per entry; -- source maps when enabled; -- a versioned strict-JSON manifest; -- content hashes or integrity values; -- declared external imports; and -- enough metadata for diagnostics and fresh generation loads. - -Writes should use a temporary output directory followed by an atomic rename so a loader never sees a half-written generation. - -### 10.3 Bundle rules - -- `@earendil-works/chord` must be externalized so a plugin uses the host's one runtime and branding symbols. -- Other dependencies are bundled by default. The explicit bundler API uses an application external allowlist; the package-level API also externalizes peer dependencies because the host provides them. -- Built-in module use may be allowed for Node entries but is not a trust or sandbox policy. -- Dynamic imports must be lowered through the loader's restricted `require`, and unresolved externals must be reported deterministically. -- Bundle output must not depend on Pi's repository path aliases. -- Rebuilding unchanged inputs should produce stable content except for documented metadata. -- Diagnostics must identify the entry and original source location. - -The concrete bundler engine is an implementation detail. Select it with a spike covering TypeScript and ESM inputs, CommonJS output, source maps, externals, content hashing, and programmatic diagnostics before adding a dependency. - -### 10.4 Bundle loading - -The bundle loader must: - -1. parse and validate the manifest; -2. select one application-requested entry; -3. verify integrity before activation when integrity is present; -4. compile it directly with `node:vm` as a fresh generation outside Node's module caches; -5. validate that its exports are plugins with unique non-empty IDs; -6. return `LoadedPlugins`; and -7. provide idempotent loader disposal. - -Plugin discovery, installation, version resolution, download, signature trust, and update policy remain application responsibilities. - -## 11. Pi migration boundary - -The following existing files describe behavior that should become Chord responsibility through a rewrite: - -| Existing area | Chord responsibility | -|---|---| -| `packages/agent/src/plugins/services/types.ts` | service tokens, modes, remote contract checks, strict JSON, snapshots, updates, connection interfaces | -| `packages/agent/src/plugins/services/replicated-state.ts` | authoritative replicated state and delivery semantics | -| `packages/agent/src/plugins/services/provider.ts` | provider classification, calls, singleton replacement, keyed generations, snapshots | -| `packages/agent/src/plugins/services/namespace.ts` | stable remote facades, hydration, state updates, keyed observation | -| `packages/coding-agent/src/experimental/facets.ts` | plugin environment, dependency ledger, lifecycle graph, host, reload | -| `packages/coding-agent/src/experimental/facet-loader.ts` | static and combined loaders plus loaded-generation ownership | -| generic service sections of `packages/protocol/src/protocol.ts` | Chord-owned versioned service/RPC envelope | - -The following must remain outside Chord: - -| Existing area | Downstream responsibility | -|---|---| -| `packages/coding-agent/src/experimental/services/connection.ts` | Pi connection state, selected-session attachment, route rebinding, Pi client adapter | -| `packages/coding-agent/src/experimental/services/server.ts` | server-wide session directory and management implementations | -| `packages/coding-agent/src/experimental/services/worker.ts` | Session worker host construction and Pi protocol publication adapter | -| `packages/server`, `packages/client`, and process managers | framing, routing, authentication, attachment, process lifecycle, reconnect policy | -| slash-command, model, account, transcript, TUI, and agent-controller services | application contracts and plugin implementations | -| `source-resolver.ts` and Pi internal process entrypoints | Pi source execution and process policy | - -The Pico5 Chord usage guide, experimental service tests, and remote plugin fixture are behavioral input. They are not normative Chord APIs. Once migration finishes, generic semantics should be documented in Chord and Pi documents should cover only their host-specific contracts and adapters. - -Migration should happen only after Chord passes its standalone conformance suite: - -1. implement Chord without changing Pi callers; -2. add thin Pi adapters and migrate generic service imports; -3. migrate the experimental plugin host and loaders; -4. adapt Pi's routed protocol to carry Chord envelopes; -5. run local, loopback, framed, keyed-generation, reload, and TUI integration tests; and -6. delete duplicate experimental generic implementations only after all consumers use Chord. - -No compatibility shim is required unless separately requested. - -## 12. Proposed source layout - -This is a planning aid, not a requirement to create all files immediately. - -```text -packages/chord/ - src/ - index.ts platform-neutral public API - api.ts root-exported functions - types.ts root-exported types, including strict JSON values - context/ - index.ts invocation context constants and functions - errors.ts lifecycle, service, and RPC errors - services/ - types.ts tokens, contracts, modes, snapshots - state.ts source and replica state primitives - state-internals.ts private replicated-state metadata - provider.ts singleton/keyed provider runtime - facade.ts stable local and remote facades - host-bindings.ts graph-facing service slots - rpc/ - peer.ts symmetric request/cancel plumbing - protocol.ts versioned generic envelopes and parsing - services.ts service protocol over a peer - loopback.ts deterministic in-memory duplex transport - plugins/ - types.ts plugin and environment types - lifecycle.ts activation and resource ownership - graph.ts validation and ordering - host.ts start, update, reload, dispose - loader.ts static and combined loaders - node/ - bundle.ts Node CommonJS bundler - bundle-loader.ts manifest validation and generation loading - test/ - ... - test-fixtures/ - bundled-plugin/ - README.md - PLANNING.md -``` - -If Node-only APIs are exported, they should use a separate package export such as `@earendil-works/chord/node` or `@earendil-works/chord/bundler`; importing the main runtime must not load Node-only modules. - -## 13. Work packages - -### WP0 — Contract decisions and test harness - -Deliver: - -- settle public vocabulary and the invocation-context shape; -- settle the admitted-call unload policy; -- settle protocol version negotiation; -- choose whether remote context position is fixed and trailing; -- create deterministic in-memory duplex and controllable race test helpers; -- add package-boundary checks preventing Pi imports; and -- add compile-only fixtures for valid and invalid remote contracts. - -Exit condition: every later work package can target explicit behavior without importing experimental implementations. - -### WP1 — Foundations and replicated state - -Deliver: - -- strict JSON type/checking; -- neutral context, cancellation, and cancellation-aware waiting; -- mutable authoritative state; -- cold replica state; -- snapshot hydration, ordered updates, clear, and rehydrate; -- borrowed immutable value contract; and -- listener error reporting. - -Tests include cold/hydrated subscriptions, immediate delivery, source update order, snapshot/update races, sequence gaps, cancellation, invalid JSON, and listener failure isolation. - -### WP2 — Symmetric RPC peer - -Deliver: - -- versioned peer envelopes and validation; -- request/response correlation; -- inbound handler dispatch; -- cancellation and disconnect behavior; -- serializable sanitized errors; -- ordered notifications; and -- loopback transport. - -Tests include requests initiated from both peers, crossed concurrent requests, duplicate/unknown IDs, pre-aborted requests, cancellation isolation, malformed messages, send failure, disconnect during calls, and handler exceptions. - -### WP3 — Service provider and facades - -Deliver: - -- service tokens and locality policy; -- singleton and keyed provider registration; -- implementation member classification; -- local service slots and stable facades; -- remote method invocation over `RpcPeer`; -- catalogues generated from provisions; -- keyed generations and stale-call fencing; and -- service error mapping. - -Tests include local and loopback paths, captured method stability, provider withdrawal/replacement, mode errors, unsupported members, strict-JSON argument/result checks, local-only isolation, keyed reuse, and concurrent callers. - -### WP4 — Remote state and service subscriptions - -Deliver: - -- complete singleton and keyed subscription snapshots; -- atomic snapshot/update buffering; -- state update publication; -- singleton unavailable/replaced events; -- keyed directory hydration and ordered reconciliation; -- observer task cancellation; and -- clear/rehydrate on connection replacement. - -Tests include late subscribers, updates racing hydration, multiple state members, provider replacement, keyed hydration before handlers, disconnect, reconnect, sequence gaps, stale frames, and observer errors. - -### WP5 — Plugin host and graph - -Deliver: - -- plugin/environment APIs; -- setup-derived ledger; -- graph validation and topological ordering; -- local and connected service resolution; -- lifecycle ownership; -- startup failure cleanup; -- host disposal; and -- static loader integration. - -Tests include consumer-before-provider setup, missing/duplicate/ambiguous providers, mode mismatch, cycles, asynchronous setup rejection, activation order, reverse cleanup, cleanup aggregation, connection hydration failure, and service access guards. - -### WP6 — Generation loading, unload, and reload - -Deliver: - -- combined loaders; -- complete structural generation replacement; -- shape-preserving targeted reload; -- stable surviving service slots; -- keyed instance retirement; -- generation fencing for late calls and updates; -- admitted-call drain/cancel policy; and -- precise failure reporting and loader disposal order. - -Tests include load failure cleanup, setup failure with old generation retained, unload with retained hard consumer rejection, provider gaps, captured local and remote methods, replacement snapshots, activation failure after cutover, disposal failure aggregation, concurrent update/dispose, and old-generation late publication. - -### WP7 — Node CommonJS bundler and bundle loader - -Deliver: - -- bundler-engine spike and decision; -- programmatic bundler API; -- versioned manifest; -- independent content-addressed entries; -- source maps and diagnostics; -- Chord externalization; -- atomic output replacement; -- manifest/integrity validation; and -- fresh VM-compiled generation loading outside Node's module caches. - -Tests bundle an application-neutral fixture with two opaque entries and a third-party dependency, load each independently, activate it, reload changed source, prove unchanged source hashes are stable, reject corrupt manifests/integrity, and prove output resolves no Pi packages. - -### WP8 — Pi adoption - -This work is downstream of Chord rather than an implementation dependency. - -Deliver: - -- Pi adapters for context metadata and routed RPC transport; -- migration of experimental service and plugin host consumers; -- preservation of selected-session and TUI behavior outside Chord; -- framed integration and reload coverage; and -- removal of superseded generic experimental code. - -Exit condition: Pi depends on Chord, while Chord remains independently packable and contains no Pi imports. - -## 14. Required conformance matrix - -The standalone suite must cover at least: - -### Plugin graph and lifecycle - -- declaration only during setup; -- setup service access rejection; -- deterministic graph ordering; -- missing, duplicate, ambiguous, and cyclic dependencies; -- activation and reverse disposal; -- startup and disposal failure aggregation; -- local and connected providers in one graph; -- complete structural load/unload; and -- shape-preserving reload. - -### Services - -- singleton and keyed modes; -- unrestricted local contracts; -- remote member classification; -- strict JSON and `void` results; -- stable local and remote facades; -- provider withdrawal and replacement; -- keyed generation fencing; -- per-call cancellation; and -- connection and host access guards. - -### Replication - -- initialized source and cold replica; -- immediate hydrated subscription delivery; -- snapshot/update race freedom; -- ordered updates and gap handling; -- disconnect clearing; -- replacement and reconnect rehydration; -- keyed state before observer startup; and -- listener/handler cleanup. - -### RPC - -- symmetric calls in both directions; -- request correlation and cancellation isolation; -- malformed and oversized-message policy; -- sanitized errors; -- disconnect cleanup; -- subscription cleanup; -- protocol-version mismatch; and -- deterministic in-memory and framed-adapter tests. - -### Loading and bundling - -- loader order and reverse disposal; -- fresh generation evaluation and garbage-collection eligibility; -- manifest validation and integrity; -- independent entries; -- dependency bundling and explicit externals; -- source maps and diagnostics; -- atomic output; and -- packed-package execution outside the monorepo. - -Race tests should control exact points rather than use timing: subscription capture versus state update, request admission versus cancellation, provider withdrawal versus invocation, instance close versus call, reload cutover versus update publication, and host dispose versus activation. - -## 15. Non-goals for the initial implementation - -- Pi host APIs or built-in Pi services; -- plugin discovery, installation, download, registry, or package resolution; -- trust policy, code signing, sandboxing, or capability security; -- network listeners, socket framing, reconnect loops, routing, or authentication; -- server/client roles or fixed process topologies; -- durable state, database integration, migrations, or transactional application writes; -- CRDTs, offline writes, or mutation replay; -- arbitrary object remoting, callbacks, remote references, or garbage collection of references; -- serialized UI trees, remote tools, or remote hooks; -- generic contribution registries; applications can expose these as process-local services; -- automatic retries of mutating calls after uncertain disconnects; and -- backward compatibility with experimental Pi APIs or wire formats. - -## 16. Decisions required before implementation - -The following decisions should be recorded in this document or small ADRs before WP1/WP2/WP7 begins: - -1. Final public names: `Plugin` versus `Facet`, `Host`, `Peer`, and `Connection`. -2. The exact neutral context API and remote method context position. -3. Whether context metadata propagation is in the first RPC envelope or added with a compatible optional field. -4. Wait-versus-cancel policy for admitted provider calls during unload. -5. Peer protocol negotiation and maximum message-size ownership. -6. State sequence-gap recovery: automatic resubscribe versus terminal binding failure. -7. Bundler engine and its runtime/development dependency placement. -8. Manifest schema, integrity algorithm, and external-module resolution contract. -9. Whether browser-compatible core behavior is an immediate tested requirement or only an architectural constraint. - -None of these decisions should introduce Pi concepts into Chord. - -## 17. Definition of done - -Chord's initial scope is complete when: - -- a standalone application unrelated to Pi can define local and remote singleton/keyed services; -- two symmetric peers can call each other, cancel calls, subscribe, disconnect, and rehydrate; -- replicated state satisfies the snapshot/update and replacement semantics above; -- plugins activate from a validated graph and own all registered resources; -- a running host can load, unload, structurally replace, and shape-preservingly reload plugin generations; -- stable service facades survive provider replacement as specified; -- the Node bundler emits and reloads independent content-addressed plugin entries; -- all race and lifecycle conformance tests pass; -- the packed package works outside the Pi monorepo; and -- an automated boundary check proves that Chord has no imports or dependencies on the rest of Pi. diff --git a/packages/chord/README.md b/packages/chord/README.md index f9677936c..db3f507db 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -1,113 +1,95 @@ # @earendil-works/chord -Chord is an application-composition runtime for systems assembled from -plugins/extensions. It provides facets, services, replicated state, and a -pluggable remote-service boundary. It is developed as a standalone package in -the Pi monorepo, but it is not a Pi package: it does not depend on any other Pi -workspace package and can be used by unrelated applications. +Chord builds applications out of **objects** that live in several processes (a server, workers, browser tabs) and can be **hot reloaded** while they run. -## What Chord is for +- An **object** is state plus code at an address. It is created on demand and lives while someone holds it. +- Everyone gets a **handle**, never the object. Handles work the same locally and across processes, and survive reloads. +- Every object knows its **principal**: an owner and a guest get different objects, with different methods. +- **State** is observable JSON. Remote viewers get a snapshot followed by deltas. +- **Hot reload** swaps code while it runs. Work in flight finishes on the code it started with, state is kept, and incompatible changes migrate at a safe point. -A single application feature may need to run in several environments: for -example, an agent worker, a terminal UI, and a remote WebUI. Chord provides the -generic machinery to write such extensions in a way that is both delightful for -humans as well as agents. +Contracts are runtime values described with [TypeBox](https://github.com/sinclairzx81/typebox) schemas. Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. -The design has a few connected pieces: +## Entry points -- **Plugins** are synchronous setup units that declare the services they provide - and require. After every plugin has declared its shape, a host validates the - complete dependency graph, binds services, activates providers before consumers, - and disposes resources in reverse dependency order. These units are called - *facets*. +| Import | Contents | Runs in | +|--------|----------|---------| +| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (`cell`, `derivation`, `listener`, `resource`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, and `replicatedState` with its types. | Node and browsers | +| `@earendil-works/chord/node` | Child host processes: `createHostManager`, `forkDriver`, `childControl`, `processLock`; the facet module loader `nodeLoader`. | Node | +| `@earendil-works/chord/context` | Explicit cancellation and invocation values: `BACKGROUND_CONTEXT`, `withCancel`, `withAbortSignal`, `withoutAbortSignal`, `withContextValue`, `awaitWithContext`. | Node and browsers | +| `@earendil-works/chord/delta` | The transactional JSON tracker, structural diff, immutable apply, and the wire encoder/decoder for operation batches. | Node and browsers | -- **Facets** are parts of a plugin. Each facet is bundled up separately and runs - in the process or environment where it's supposed to run. You can use facets - to split a plugin into separate pieces that need to be loaded into different - processes and environments (think backend, browser, TUI etc.) +## Quick start -- **Services** are typed, stable tokens with either one provider (**singleton**) - or dynamic keyed instances (**keyed**). A service can be process-local, with - an unrestricted JavaScript contract, or remotely exposable. Consumers retain a - stable facade while a provider disconnects or is replaced. +```ts +import { Type } from "typebox"; +import { cell, createHost, method, remote } from "@earendil-works/chord"; -- **Replicated state** exposes authoritative state to local and remote - connected consumers. Producers publish atomic overlay transactions with - `change(context, callback)`; consumers receive complete immutable values. Draft - proxies exist only during the callback and become unusable afterward. Preparation - materializes a structurally shared immutable candidate and one exact decoded - operation batch, while each remote client/state stream owns independent path-codec - state. Replicas become unready on disconnect or replacement until rehydrated. +// contract.ts: shared by provider and consumers +export const Counter = remote("demo.counter", { + version: 1, + input: Type.Object({ name: Type.String() }), + variants: { + default: { + state: { count: Type.Number() }, + methods: { increment: method(Type.Object({ by: Type.Number() }), Type.Null()) }, + }, + }, +}); -- **Delta tracking** records and coalesces operations over tracked plain JSON. - It preserves common string and array operations, supports durable base - batches, and validates untrusted operations as they are applied. Batches - guarantee convergence but are not canonical or necessarily minimal. +const host = createHost({ name: "server", principalKey: (p) => p.kind }); -- **Remote service sources** advertise services available outside a facet host - and open bindings for the services its facets require. Bindings carry logical - calls and subscriptions through an application-supplied adapter. Chord - requires strict-JSON arguments, results, snapshots, updates, and catalogues, - but does not prescribe framing, routing, transport, or an application wire - envelope. `JsonRepresentation` derives a wire-safe type for application data - with unknown payloads, while `isJsonValue()` validates received values at an - adapter boundary. Symmetric RPC peers are planned as one optional - implementation of this boundary. +host.serve(Counter, ({ name }, _principal, self) => { + const state = self.state({ count: cell(Type.Number(), 0) }); + return { + kind: "default", + count: state.count, + increment: async ({ by }, ctx) => { + state.count.change(ctx, (n) => n + by); + return null; + }, + }; +}); -- **Context** Chord provides a Go-like context system for cancellation and - invocation-scoped application values. Applications can carry permissions or - telemetry through those values without Chord depending on either. +await host.root(async (ctx) => { + const clicks = ctx.get(Counter, { name: "clicks" }); // a handle; held until this root ends + clicks.count.subscribe((n) => console.log("count", n)); + await clicks.increment({ by: 2 }, ctx); // logs 0, then 2 +}); +``` -The current runtime exports service tokens, singleton and keyed providers, -remote bindings, replicated state, facet hosts, and facet loaders from -`@earendil-works/chord`. Import public types and general runtime APIs from the -package root. Context constants and functions live in -`@earendil-works/chord/context` because their generic names should not pollute -the root API. -Chord-owned identifiers use the `chord.*` namespace and its reserved service -prefix is `$chord.*`. +## Documentation -## Remote service adapters +- [Tutorial](docs/tutorial.md): a shared todo list, built up section by section. Every example runs as `test/tutorial.test.ts`. +- [Usage guide](docs/guide.md): API reference for contracts, objects, state, roots, lifetime, plugins, hot reload, hosts, and links. +- [Specification](docs/spec.md): compatibility, lifetime, and reload semantics. +- [Implementation notes](docs/notes.md) and [requests](docs/requests.md): decisions, review findings, and resolved requests. +- [Delta guide](src/delta/README.md): ownership, lifecycle, operation, and replica contracts of `@earendil-works/chord/delta`. -Chord owns its transport-independent service wire grammar. Consumer adapters -use `createServiceCatalogueCall()`, `createServiceSubscribeCall()`, and -`createServiceUnsubscribeCall()` for `$chord.service` control calls. -`createRemoteServiceEndpoint()` handles those calls for one provider consumer, -including subscription activation and cleanup. `parseServiceCall()`, -`parseServiceCatalogue()`, and the decoded/wire snapshot and update parsers -validate Chord semantics after an adapter has established a strict-JSON -boundary. `RemoteServiceErrorCode` and `REMOTE_SERVICE_ERROR_CODES` define the -service errors that may cross that boundary. +## Replicated state -Replicated state operations use one `createServiceStateEncoder()` at the -provider side and one `createServiceStateDecoder()` at the consumer side for -each subscription. Those registries create an independent Delta path dictionary -for every instance/member state and reset it on replacement, unavailability, -close, or fresh hydration. Applications may place these values inside any -routing, request, response, or event envelope; Chord does not prescribe that -outer protocol. +`replicatedState` is a standalone observable immutable JSON value, independent of hosts. Durable uses it for document, view, and task-graph states. -Provider subscriptions atomically capture a snapshot and buffer subsequent -updates until activation. Activation and reentrant publication use the same FIFO. -The provider retains at most 100 pending updates per subscription, excluding the -running delivery. Adding update 101 replaces the entire pending queue with -`{ type: "reset", snapshot }`: a current full subscription snapshot whose state -members contain `[["r", value]]` and their new sequence baselines. It also captures -current instance membership, so discarded spawn/close/replacement events cannot -leave stale instances behind. Live keyed generations retain their existing handles. -The reset carries the context of the publication that triggered overflow. +```ts +import { replicatedState } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -Consumers and codecs must handle this explicit reset before accepting subsequent -ordinary deltas. A root operation in an ordinary `state` update does **not** permit -a sequence gap. Resets restart the subscription's path dictionaries, and later -deltas must be contiguous from each reset baseline. Transport adapters must -preserve snapshot/reset/update order; they must not drop encoded delta batches or -assume their own asynchronous queues are bounded by the provider's queue. +const status = replicatedState({ output: "", count: 0 }); // takes ownership of the initial value +status.subscribe((value, context, delivery) => console.log(delivery.kind, delivery.sequence, value.count)); +status.change(BACKGROUND_CONTEXT, (draft) => { + draft.output += "done\n"; + draft.count += 1; +}); +``` + +`replicatedState(source, { onError })` instead attaches a publication-only state to an authoritative revision source (`ReplicatedStateSource`): it captures a snapshot, then publishes every committed frame in cursor order. + +A successful `change()` publishes exactly one atomic revision. If its callback throws, the value and sequence stay unchanged. Draft handles become unusable when the callback returns. Values placed through drafts are copied; `replicatedState(initial)` and `replace(context, value)` take ownership of alias-free strict-JSON roots without copying them, so callers must not mutate them afterwards. Published values are not frozen. + +Each subscription serializes its callbacks: hydration settles before updates start. At most 100 deliveries wait behind a running callback; on overflow only the newest pending value is kept, so update sequences may skip. Listener failures go to `onError` (attached states) or are thrown in a microtask (mutable states), and never stop other subscriptions. Unsubscribing discards pending work without aborting the running callback. ## Tracking JSON deltas -Import the standalone transactional tracker from `@earendil-works/chord/delta`: - ```ts import { applyImmutable, track } from "@earendil-works/chord/delta"; @@ -121,144 +103,4 @@ tracker.adopt(prepared); const replica = applyImmutable(prepared.base, prepared.ops); ``` -`tracker.value` is always the latest adopted immutable revision. Preparation does -not change authority; adoption validates the preparation and swaps the root pointer. -Assigned containers are copied by value and unchanged subtrees may be shared between -revisions. - -The tracker uses trusted immutable ownership rather than defensive copying or -freezing. `track(initial)`, `prepareReplace(value)`, `replicatedState(initial)`, and -`replace(context, value)` take ownership of alias-free strict-JSON roots without -walking them. Callers must not mutate transferred or published data. Values placed -through drafts are already copied, so that walk also rejects non-strict JSON before -the draft changes. Published values are not frozen. In-process loopback consumers -may share their containers with the provider; mutating a consumed value violates -the contract and can corrupt authority. Clone or serialize at any mutable trust -boundary. - -Replicated state provides the same model through a callback: - -```ts -const initial = { output: "", count: 0 }; -const status = env.replicatedState(initial); // transfers ownership of initial -status.change(context, (draft) => { - draft.output += "done\n"; - draft.count += 1; -}); -``` - -A successful `change()` publishes exactly one atomic revision. If its callback -throws, the original value and sequence remain unchanged. Draft handles become -unusable when the callback returns. Chord emits string append and front-truncate -operations, array splices and permutations, sets, and deletes; large edit sets may -fold into a complete replacement. Remote connection plumbing encodes each batch -independently for every client/state pairing. - -### Public state subscriptions - -`state.subscribe(async (value, context, delivery) => { ... })` serializes callbacks -independently for each subscription. Hydration starts immediately when a value is -available, and its returned promise must settle before updates start. Synchronous -callbacks still run synchronously. Read the captured `value` inside an asynchronous -callback: `state.value` may already refer to a later revision. - -Each subscription retains at most 100 pending complete-value deliveries, excluding -the running callback. On overflow, only the newest pending value, context, and -delivery metadata are retained; a not-yet-started initial hydration is preserved. -Thus public delivery sequences may skip. This is a frame-count policy, not a byte -limit. Internal exact-operation subscriptions remain synchronous and receive every -revision, independently of slow public callbacks. - -The returned unsubscribe function immediately discards pending work and prevents -new callbacks. It neither aborts nor waits for the running callback. Synchronous -throws and promise rejections are observed, reported, and do not stop other -subscriptions or subsequent callbacks. Attached sources and remote bindings use -their `onError` handler; local mutable states (and attached sources without a -handler) report failures by throwing in a microtask. Unsubscribing does not hide a -later rejection from the running callback. - -The standalone [Delta guide](src/delta/README.md) defines the complete ownership, -lifecycle, operation, and replica contracts. - -## Bundling and loading facets - -`@earendil-works/chord/bundler` uses esbuild to turn ESM or TypeScript application -entries into independent, content-addressed CommonJS files. The package-level API -reads plugin identity and build configuration from `package.json`, then applies -facet path conventions supplied by the host application: - -```json -{ - "name": "@example/my-plugin", - "version": "1.0.0", - "type": "module", - "peerDependencies": { - "@earendil-works/chord": "^0.84.4" - }, - "chord": { - "facets": { - "worker": "./src/custom-worker.ts", - "presentation": false - } - } -} -``` - -```ts -import { bundleFacetPackage } from "@earendil-works/chord/bundler"; - -await bundleFacetPackage({ - packagePath: "/path/to/my-plugin", - outdir: "/application-owned/plugin-builds/my-plugin", - defaultFacets: { - worker: "src/worker.ts", - presentation: "src/presentation.ts", - }, -}); -``` - -Existing conventional files become entries unless `chord.facets` overrides or -disables them. Peer dependencies are externalized and resolved against the host -when loading. Chord never installs dependencies or runs package lifecycle -scripts. `bundleFacets()` remains available as the lower-level API for callers -that already have explicit plugin identity and entry mappings. - -The output directory contains one `.cjs` file per entry plus -`chord-facets.json`. Load one application-selected entry through the Node-only -loader: - -```ts -import { createFacetBundleLoader } from "@earendil-works/chord/node"; - -const loader = createFacetBundleLoader({ - manifestPath: "/application-owned/plugin-builds/my-plugin/chord-facets.json", - entry: "worker", - resolveExternal: (specifier) => import.meta.resolve(specifier), -}); -const loaded = await loader.load(); -``` - -Each `load()` verifies SHA-256 integrity and compiles the CommonJS body directly -with `node:vm` instead of putting the plugin into Node's CommonJS or ESM module -cache. Externals are resolved by the host and loaded through a restricted -`require`; esbuild lowers dynamic imports so they use the same path. Disposing a -retired generation releases the loader's facet references, making its compiled -code eligible for garbage collection once plugin-owned resources are also gone. - -For transport to another Node host, `readFacetBundleArtifact()` packages one -verified manifest entry with its source, and `createFacetBundleArtifactLoader()` -materializes fresh temporary generations while resolving externals against the -receiving host. - -To reload, load a candidate, pass its facets to `FacetHost.reload()`, dispose the -candidate on failure, and dispose the retired `LoadedFacets` only after a -successful cutover. The host activates and validates the candidate while the -currently active providers remain routed, then replaces each singleton directly -without an unavailable interval. Stable service handles therefore do not become -disconnected during an ordinary reload. Keyed instances -remain incarnation-specific and replacements receive fresh generations. The -bundler writes a complete temporary directory before replacing the previous -output, so loaders do not observe partially built generations. - -See [PLANNING.md](PLANNING.md) for the broader RPC and generation-loading -architecture. +`tracker.value` is always the latest adopted immutable revision. Preparation does not change authority; adoption validates the preparation and swaps the root pointer. Assigned containers are copied by value and unchanged subtrees may be shared between revisions. diff --git a/packages/chord/docs/guide.md b/packages/chord/docs/guide.md new file mode 100644 index 000000000..1adb8381b --- /dev/null +++ b/packages/chord/docs/guide.md @@ -0,0 +1,750 @@ +# Chord usage guide + +> **Status:** describes the design in [spec.md](spec.md) (v7). The implementation in `src` is in progress; names may still change. + +Chord builds applications out of **objects** that live in several processes (a server, workers, browser tabs) and can be **hot reloaded** while they run. + +- An **object** is state plus code at an address. It is created on demand and lives while someone holds it. +- Everyone gets a **handle**, never the object. Handles work the same locally and across processes, and survive reloads. +- Every object knows its **principal**: an owner and a guest get different objects, with different methods. +- **State** is observable JSON. Remote viewers get a snapshot followed by deltas. +- **Hot reload** swaps code while it runs. Work in flight finishes on the code it started with, state is kept, and incompatible changes migrate at a safe point. + +## Contents + +- [Quick start](#quick-start) +- [Contracts](#contracts) +- [Serving objects](#serving-objects) +- [Declarations](#declarations) +- [Activation](#activation) +- [Principals and variants](#principals-and-variants) +- [Handles](#handles) +- [State](#state) +- [Calls, ctx and roots](#calls-ctx-and-roots) +- [Lifetime](#lifetime) +- [Registries](#registries) +- [Plugins](#plugins) +- [Hot reload](#hot-reload) +- [Hosts and links](#hosts-and-links) +- [Durable](#durable) +- [Errors](#errors) +- [Rules of thumb](#rules-of-thumb) + +## Quick start + +```ts +import { Type } from "typebox"; +import { cell, createHost, method, remote } from "@earendil-works/chord"; + +// contract.ts: shared by provider and consumers +export const Counter = remote("demo.counter", { + version: 1, + input: Type.Object({ name: Type.String() }), + variants: { + default: { + state: { count: Type.Number() }, + methods: { increment: method(Type.Object({ by: Type.Number() }), Type.Null()) }, + }, + }, +}); + +const host = createHost({ name: "server", principalKey: (p) => p.kind }); + +host.serve(Counter, ({ name }, _principal, self) => { + const state = self.state({ count: cell(Type.Number(), 0) }); + return { + kind: "default", + count: state.count, + increment: async ({ by }, ctx) => { + state.count.change(ctx, (n) => n + by); + return null; + }, + }; +}); + +await host.root(async (ctx) => { + const clicks = ctx.get(Counter, { name: "clicks" }); // a handle; held until this root ends + clicks.count.subscribe((n) => console.log("count", n)); + await clicks.increment({ by: 2 }, ctx); // logs 0, then 2 +}); +``` + +## Contracts + +A contract is a runtime value: a name, a version number, and a description of the members. Shipped bundles have no TypeScript, so everything the runtime checks must be in the contract. TypeScript types are inferred from it. + +### Remote types + +Remote types can be reached from other hosts. Everything that crosses the wire has a schema, and the provider validates it. + +```ts +export const Chat = remote("pi.chat", { + version: 1, + input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + variants: { + viewer: { + state: { view: ConversationViewSchema }, + }, + participant: { + state: { view: ConversationViewSchema }, + methods: { + send: method(SendInputSchema, Type.Object({ submissionId: Type.Number() })), + stop: method(Type.Null(), Type.Null()), + }, + }, + }, +}); +``` + +### Local types + +Local types are reachable only inside their host. Their members may be anything: functions, callbacks, native objects. Non-JSON members get opaque or function descriptors. + +```ts +export const Index = local("pi.index", { + version: 1, + input: Type.Object({ session: Type.String() }), + members: { + search: method(Type.Object({ query: Type.String() }), Type.Array(HitSchema)), + onChange: method(Type.Function([ChangeSchema], Type.Void()), UnsubscribeSchema), + harness: opaque("pi.durable-harness@1"), + }, +}); +``` + +### Inputs identify objects + +`{ session: "S1", conversation: 1 }` always means the same object. Inputs are normalized (defaults applied, unknown fields removed), validated, and serialized canonically. Fields equal to their default are left out, so adding a new optional field with a default doesn't change existing addresses. Changing a default, or how inputs are normalized, needs a new type name. + +### Versions and compatibility + +Every bundle carries its own copy of each contract it was built against. When a consumer binds to a provider, the two copies are compared: + +> A consumer built against version M binds to a provider at version N if **N = M and the provider's description covers the consumer's**. Otherwise the provider must offer an adapter in `accepts[M]`, or binding fails with `contract_mismatch`. + +"Covers" means the provider still offers everything the consumer was built against: +- every member exists with the same kind; +- methods accept everything the consumer can send (new arguments must be optional) and return at least what it expects (extra result fields are fine); +- state values fit the consumer's schema; +- opaque members have the same name and version. + +| Change | Version | Result | +| --- | --- | --- | +| Add a method, an optional argument, a result field, a state field | keep | binds | +| Remove or rename a member; make an argument required; change a type | **bump** | old consumers need `accepts`, or must be reloaded too | +| Same shape, different meaning (`query` is now a regular expression) | **bump** | as above; undetectable, so it's on you | +| Forgot to bump on a breaking shape change | keep | refused: "consumer needs member `search`" | + +```ts +// version 2 changed what `search` means; old consumers keep working through an adapter +host.serve(Index, indexFactory, { + accepts: { + 1: (current) => ({ search: ({ query }, ctx) => current.search({ query: escapeRegExp(query) }, ctx) }), + }, +}); +``` + +## Serving objects + +```ts +host.serve(Type, (input, principal, self) => behavior, options); +``` + +- The **factory** runs synchronously. It declares what the object needs and returns its behavior: methods and state members, plus `kind` for remote types. Returning a promise throws. +- There is one **record** per `(type, input, principal)`. It holds the object's state, resources and holds, and survives reloads. Only the behavior is replaced. +- **Options:** + - `state: { version, migrate }`: the state version and its migrations (see [Hot reload](#hot-reload)); + - `accepts`: adapters for consumers built against older contract versions; + - `linger`: how long to keep the object after the last hold is released; + - `eager`: activate when the plugin loads, and keep it for the host's lifetime. + +## Declarations + +A reload runs the factory again over the same record. Declarations are matched by **property name**, so the new code's `status` cell is the old code's `status` cell, with its value. + +```ts +host.serve(StatusStore, ({ session }, _host, self) => { + const state = self.state({ + status: cell(StatusSchema, { running: false, tasks: 0 }), + errors: cell(Type.Array(Type.String()), []), + }); + const catalog = self.get(Catalog, {}); + const res = self.resources({ + db: resource({ + rev: 1, + open: async ({ signal }) => openSqlite(statusPath(session), { signal }), + close: async (db) => db.close(), + }), + }); + self.listen({ + catalogChanges: listener(catalog.rows, (rows, ctx) => state.errors.replace(ctx, checkRows(rows))), + }); + self.tasks({ + flush: async (ctx) => { + while (!ctx.signal.aborted) { + await sleep(10_000, ctx.signal); + await res.db.value.run("INSERT INTO history VALUES (?)", [JSON.stringify(state.status.value)]); + } + }, + }); + const derived = self.derive({ + summary: derivation([state.status, state.errors], (s, e) => ({ ...s, failing: e.length > 0 })), + }); + return { summary: derived.summary, report: reportMethod(state) }; +}); +``` + +| Declaration | What it is | On reload | +| --- | --- | --- | +| `self.state({ name: cell(schema, initial) })` | an observable JSON cell; the schema is required | same cell and value; migrated when the state version is bumped | +| `self.get(Type, input, { at? })` | a hold on another object; its identity is the address | kept if declared again, otherwise released | +| `self.resources({ name: resource({ rev, deps?, exclusive?, open, close }) })` | a connection, watch, socket, port | kept; reopened only if `rev` or `deps` change | +| `self.listen({ name: listener(source, fn) })` or `listener(source, select, fn)` | reacts to a state or resource event | the same subscription, with the new `fn`, as long as the source (resource) and `select` are unchanged | +| `self.tasks({ name: async (ctx) => ... })` | long-running background work | the old run is aborted through `ctx.signal`, and the new code starts | +| `self.derive({ name: derivation(sources, fn) })` | live projection of state; `fn` is synchronous | kept; `fn` replaced | +| `self.register(Registry, { key: value })` | an entry in a registry | replaced at commit | +| `self.expose(res.name)` | a member backed by a resource (local types) | follows the resource | + +A listener keeps its subscription across reloads when it is declared on the same resource (or source) with the same `select`: the same function, or one with the same source text. `select` is not called again, so a source that accepts a single subscriber (a Durable watch) survives reloads, and `select` may wrap the value in a new adapter object. A `select` whose result depends on captured values that change between reloads is not noticed; give such a listener a new name. + +A name you no longer declare is released, closed or stopped, after the old code has finished what it was doing. + +**State that must survive a reload lives in cells,** never in closure variables. A closure variable lasts as long as one version of the code. + +```ts +// wrong: reset on every reload +host.serve(Visits, (_, _p, self) => { + let visits = 0; + return { hit: async (_, ctx) => ++visits }; +}); + +// right +host.serve(Visits, (_, _p, self) => { + const state = self.state({ visits: cell(Type.Number(), 0) }); + return { + hit: async (_, ctx) => { + state.visits.change(ctx, (n) => n + 1); + return state.visits.value; + }, + }; +}); +``` + +## Activation + +The factory is synchronous, but activation isn't. `self.get` and `self.resources` only *declare*; the slow work happens afterwards: + +```text +get(Index, { session: "S1" }) + 1. run the factory → declarations + 2. open declared resources → e.g. connect to SQLite + 3. wait for declared holds → dependencies activate first + 4. live → handles become "live" +``` + +```ts +host.serve(Index, ({ session }, _host, self) => { + const res = self.resources({ + db: resource({ rev: 1, open: () => openSqlite(indexPath(session)), close: (db) => db.close() }), + }); + return { + search: async ({ query }, ctx) => res.db.value.all("SELECT entryId, text FROM fts WHERE fts MATCH ?", [query]), + }; +}); + +host.serve(Chat, ({ session, conversation }, principal, self) => { + const index = self.get(Index, { session }); // a placeholder until Index is live + return { + kind: "participant", + find: async ({ query }, ctx) => index.search({ query }, ctx), // only ever runs once Chat (and Index) are live + // ... + }; +}); +``` + +**Rules:** +- Inside the factory, never use what you declared: `res.db.value` throws `not_ready`, and calling a handle throws. Use them in methods, listeners and tasks. +- **Concurrent `get`s** of one address share one activation, so the database opens once. +- **Failure propagates:** if `openSqlite` throws, `Index` fails, `Chat` fails with that cause, and every waiting `get` rejects. The next `get` tries again. +- **Abandoned activations:** if everyone releases before activation finishes, the result is disposed. + +### Startup readiness and parent admission + +`await host.load(facets)` and `await host.install(facet)` wait for the eager objects they install to become live: +resources are open, declared holds are live, and registrations are published. All facets in a load are installed +before activation is awaited, so an eager consumer can use a provider in a later facet. Ordinary lazy objects stay +lazy unless an eager object holds them. Factory, dependency and resource errors reject startup with the handle's +activation error; eager failures also reach `onError`. Installation is not rolled back on failure: close the host +when abandoning startup. `install` registers synchronously, so setup/registration errors still throw synchronously. + +`await host.ready()` waits for the eager holds present when called, including direct `host.serve(..., { eager })` +and previously deferred loads. It is an activation barrier, not a continuous health check, a wait for tasks to +finish, or a remote-state hydration barrier. Later installations require another readiness wait. + +A worker may need to open its local Harness and announce control readiness **before** the parent accepts its link. +An eager reporter waiting for the server's SessionStatus cannot become live before that acceptance. Awaiting the +whole load before announcing control readiness would deadlock. Defer the whole eager barrier explicitly: + +```ts +// Connect the upstream link first; its outgoing holds can queue until the parent accepts it. +await worker.load(workerFacets, { awaitEager: false }); // also supported by install +const harness = worker.get(Harness, {}); +await harness.ready; // local bootstrap only; no upstream dependency +announceControlReady(); // parent can now accept the worker link +await worker.ready(); // full eager readiness, including upstream holds +harness.release(); +``` + +`awaitEager: false` still starts and host-holds every eager object; it skips waiting, not activation. There is no +automatic classification of upstream dependencies. Choose a local bootstrap handle with no upstream dependency, +connect before loading upstream consumers, and let the parent admit the link before the full barrier. Control +readiness means admission is possible, not that every eager object is live. Keep worker bootstrap counted as busy +until the full barrier settles, and report a later failure through the worker's fatal/startup-error path. + +To hand out a native object a resource opens (a local type's `opaque` member), expose the resource. Handles read its value once the object is live, and `undefined` before: + +```ts +host.serve(Harness, (_, _host, self) => { + const res = self.resources({ durable: resource({ rev: 1, open: () => DurableHarness.open(storage), close: (h) => h.close() }) }); + return { harness: self.expose(res.durable) }; // contract member: harness: opaque("pi.durable-harness@1") +}); + +const harness = ctx.get(Harness, {}); +await harness.ready; +harness.harness.resume(); // the DurableHarness itself +``` + +## Principals and variants + +The application defines its principals and how they're keyed: + +```ts +type Principal = + | { kind: "owner"; via: "device" | "tunnel" | "relay" } + | { kind: "guest"; id: string; name: string; session: string; mode: "view" | "full"; revision: number } + | { kind: "worker"; session: string; generation: number }; + +const host = createHost({ + name: "server", + principalKey: (p: Principal) => + p.kind === "owner" ? `owner@${p.via}` : + p.kind === "guest" ? `guest:${p.id}:r${p.revision}` : + `worker:${p.session}:${p.generation}`, +}); +``` + +- There is one record per principal key: two tabs of the owner share an object, and a guest gets its own. +- Code inside a host acts as the built-in `host` principal, so internal objects are shared. +- A link stamps the principal of whatever connects to it. Callers can never supply their own. + +**A remote factory picks a variant:** + +```ts +host.serve(Chat, ({ session, conversation }, principal, self) => { + if (!canView(principal, session)) throw forbidden(); + const c = self.get(Conversation, { session, conversation }); // as host: one shared instance + if (!canSend(principal)) return { kind: "viewer", view: c.view }; + const author = authorOf(principal); + return { + kind: "participant", + view: c.view, + send: async (input, ctx) => c.submit(input, author, ctx), + stop: async (_, ctx) => { + await c.stop(author, ctx); + return null; + }, + }; +}); +``` + +```ts +const chat = ctx.get(Chat, { session: "S1", conversation: 1 }, { at: "session:S1" }); +await chat.ready; +if (chat.kind === "participant") showComposer(chat.send); +else showReadOnly(); +``` + +**What a variant doesn't do:** +- **It isn't complete authorization.** Methods still check their arguments: that a submission belongs to this conversation, or a session ID to this principal. +- **It doesn't cover flags that change while you hold the object.** Those are state, for example `you: { canDraw: boolean }`. +- **Revocation** is `host.invalidatePrincipal(match)`. It force-disposes matching records, here and over links, and their handles become `gone`. A new `get` runs the factory with the new grants. + +Local types have no variants. Their only callers are inside the host, acting as `host`. If a local object needs to know who's acting, pass that explicitly, like `author` above. + +## Handles + +```ts +const chat = ctx.get(Chat, input, { at: "session:S1" }); +chat.status; // "loading" | "live" | "released" | "gone" +await chat.ready; // live, or rejects with the activation error +chat.kind; // the current variant +chat.shape; // { kind, members, contract } +const send = chat.send; // a stable function: always dispatches to the current code +chat.view.value; // undefined until hydrated (remote: while someone subscribes) +chat.view.subscribe((v) => render(v)); +chat.onStatus((status, error) => { if (status === "gone") reconnect(error); }); +chat.release(); +``` + +- **Calls on a `loading` handle wait** until it's live, respecting `ctx` cancellation and a deadline. +- A member missing from the current variant rejects with `not_in_variant`. +- After `release()`, every member rejects with `released`. +- **`gone`** means the object was forced away while held: revocation, archive, its host dying. Call `get` again, or show "unavailable". `onStatus` reports every status change (`live`, `loading` while a link reconnects, `released`, `gone` with the error), so a UI needn't poll. +- **A remote state member is hydrated only while subscribed.** When its last subscriber leaves, its `value` goes back to `undefined`, so a later subscriber never sees a stale value: it gets `undefined`, then the current snapshot. +- **Return values are values.** Anything long-lived you want to share is an object you reach with `get`. + +## State + +```ts +const state = self.state({ todos: cell(Type.Array(TodoSchema), []) }); + +state.todos.change(ctx, (draft) => { + draft.push({ id: "t1", text: "Write docs", done: false }); +}); +state.todos.change(ctx, (draft) => { + draft[0].done = true; +}); +state.todos.replace(ctx, []); + +state.todos.value; // immutable +state.todos.subscribe((value) => render(value)); + +// mirroring an external immutable store: subscribers get a diff, not the whole value +state.view.replace(ctx, conversation.view); // diffs when the new value reuses old containers +state.view.replace(ctx, rebuilt, { diff: true }); // always diff +``` + +- A `change` is atomic: one revision, or nothing if the callback throws. +- `change` only touches the paths you mutate, so fields your code doesn't know about survive. Use `replace` only where your code owns the whole value. +- Values are immutable. Never mutate what you read or receive. +- Remote subscribers get a snapshot, then deltas. After a reconnect or a reload they get a `reset` snapshot, and nothing is replayed. +- `replace` sends a structural diff when the new value shares containers with the old one (a new revision of the same state), and a single replacement otherwise. `{ diff: true | false }` forces either. The value is the same either way. +- A behavior may expose another object's state member as its own: `return { kind: "viewer", view: conversation.view }`. It reads `undefined` only while that object is loading, which never happens once this object is live. + +```ts +const derived = self.derive({ + visible: derivation([catalog.rows], (rows) => rows.filter((r) => visibleTo(principal, r))), + openCount: derivation([state.todos], (t) => t.filter((x) => !x.done).length), +}); +``` + +`derivation` functions are synchronous. + +## Calls, ctx and roots + +Every method gets `(args, ctx)`. Pass `ctx` on to everything you call: + +```ts +send: async (input, ctx) => { + ctx.signal.throwIfAborted(); + const uploads = ctx.get(Uploads, {}); // a hold owned by this call; released when it settles + const files = await uploads.resolve({ ids: input.uploads }, ctx); + return c.submit({ ...input, files }, author, ctx); +}, +``` + +A **root** is a short tree of calls, started at ingress: +- by a link, for a request that arrives without one; +- by `host.root(fn)`, for timers and scripts; +- by each event delivered to a `listener`; +- by each Chord call made from a `task` or a Durable phase. + +A root runs on the code that was current when it started (its **epoch**), across hosts, until it ends. Keep roots short. Long-running work is a `task`, and long waits are state. + +```ts +// wrong: holds a root (and its version of the code) open for minutes +approve: async (_, ctx) => waitForHuman(ctx), + +// right: record the request as state; the answer is a separate call +request: async (req, ctx) => { + state.pending.change(ctx, (d) => { + d.push(req); + }); + return null; +}, +answer: async ({ id, value }, ctx) => { + state.pending.change(ctx, (d) => { + settle(d, id, value); + }); + return null; +}, +``` + +## Lifetime + +An object lives while it's **held**, and every hold has an owner: + +| Owner | Acquired with | Released when | +| --- | --- | --- | +| another object | `self.get` | that object is disposed, or no longer declares it | +| a call | `ctx.get` | the call settles | +| a connection | a remote host's `get` | release, disconnect, revocation | +| the host | `eager: true` | host shutdown | +| UI | `useObject` | unmount | + +- **Disposal** runs after the last hold, plus `linger`. Tasks are aborted, listeners removed, resources closed in reverse order (every close runs, and errors are collected), then the object's own holds are released, which can dispose further objects. +- **A resource's `close` may still use the objects its record holds.** That holds for host shutdown and revocation too: `host.close()` and `invalidatePrincipal` dispose holders before what they hold, and close links last. Bound remote calls in `close` with a signal: a call over a link that is already down waits for a reconnect. +- **Forced disposal** (revocation, archive) makes handles `gone` at once and aborts in-flight calls. +- **Holds must not form cycles.** Inside one host this is checked at `get`. + +```ts +host.serve(Conversation, conversationFactory, { linger: 30_000 }); // switching back quickly doesn't reopen its watch +``` + +## Registries + +```ts +export const Routes = defineRegistry("http.routes", { conflict: "error" }); + +// contribute (only from eager objects) +host.serve(Downloads, (_, _host, self) => { + const blobs = self.get(Blobs, {}); + self.register(Routes, { + "GET /download": async (request, who, ctx) => { + const file = await blobs.find({ id: request.query.id }, ctx); + if (!canView(who, file.session)) return forbidden(); + return stream(file, ctx); + }, + }); + return {}; +}, { eager: true }); + +// consume: a request sees the table of its own epoch +server.on("request", (req, res) => + host.root(async (ctx) => { + const route = host.registry(Routes).get(`${req.method} ${pathOf(req)}`, ctx); + return route ? route(req, principalOf(req), ctx) : notFound(res); + }), +); +``` + +- Each epoch has one complete table. A reload never leaves a moment where an entry is missing. +- A request that started before a reload keeps using the old table. +- Order is first-registration order and survives replacement. +- `conflict: "error"` makes a duplicate key fail the registration (or the reload). + +## Plugins + +A plugin is a feature with one **facet** per host kind: + +```text +plugins/chat/ + contract.ts Chat, Conversation, Drafts + server.ts facet for the server + worker.ts facet for session workers + web.tsx facet for browsers +``` + +```ts +// plugins/chat/worker.ts +import { defineFacet } from "@earendil-works/chord"; +import { Chat, Conversation } from "./contract.ts"; + +export default defineFacet((host) => { + host.serve(Conversation, conversationFactory, { linger: 30_000 }); + host.serve(Chat, chatFactory); +}); +``` + +```ts +// a host loads the plugins' facets for its kind +await host.load([ + new URL("../plugins/chat/worker.ts", import.meta.url), + new URL("../plugins/status/worker.ts", import.meta.url), +]); +``` + +Built-in plugins are trusted code from this repository and load directly from source. + +The facet's `setup` argument is its host, scoped to what a plugin needs: `serve`, `name`, `root(fn)` and `registry(R)`. Code that is not a Chord method, listener or task (Durable tools and hooks, callbacks of native libraries) keeps it to start roots and read registries, instead of reaching for a process-global host: + +```ts +export default defineFacet((host) => { + host.serve(Sessions, sessionsFactory); + tools.register("rename_session", (args) => + host.root((ctx) => ctx.get(Sessions, {}, { at: "server" }).rename(args, ctx)), + ); +}); +``` + +## Hot reload + +```ts +await host.reload([new URL("../plugins/chat/worker.ts", import.meta.url)]); // or reload every changed plugin +``` + +### What happens + +1. **Load** the new modules. +2. **Prepare:** run the new factories over the existing records, without side effects. Cell writes throw, listeners and tasks don't start, registrations are collected, and changed resources are opened next to the old ones. Contracts are checked against every live consumer. Any failure leaves everything as it was. +3. **Gate:** only if a state version was bumped or an exclusive resource changes. +4. **Commit:** in one synchronous step, install the new code, swap registry tables, restart tasks, rebind listeners, and send `reset` and shape updates to subscribers. +5. **Retire:** old code keeps running the roots it started. Once they end, it's dropped, along with whatever it alone declared. + +### Compatible changes: no waiting + +Bug fixes, new methods, new cells, new resources and registration changes all switch immediately. A root that started on the old code finishes on it, and new roots get the new code. + +### Changing what state looks like or means: bump the state version + +```ts +// v1: { running, tasks } where tasks counted phases +// v2: tasks counts tasks, and usage is new +host.serve(StatusStore, ({ session }, _host, self) => { + const state = self.state({ status: cell(StatusV2, { running: false, tasks: 0, usage: emptyUsage() }) }); + return { + status: state.status, + report: async (s, ctx) => { + state.status.replace(ctx, s); + return null; + }, + }; +}, { + state: { + version: 2, + migrate: { + status: (old, from) => (from < 2 ? { running: old.running, tasks: 0, usage: emptyUsage() } : old), + }, + }, +}); +``` + +At the gate, for each live `StatusStore`: +1. **Roots already inside finish.** Their re-entrant calls are let in. +2. **Old roots that haven't touched it yet get `upgrade_conflict`** instead of old code over migrated state. They retry as a new root. +3. **`migrate` runs** on a private copy, and the result is validated against the new schema. +4. **Commit.** New roots use the new code and the migrated state. + +If the roots inside don't finish within the deadline (5 s), the reload is **aborted** and nothing changes. The error names the blocking roots. + +The runtime compares each cell's schema between old and new code: +- unchanged: compatible; +- changed with a bump: migrate; +- changed **without** a bump: the reload is refused. + +A change of meaning with an identical schema needs a bump too. Nothing can detect that one for you. + +### Resources + +```ts +const res = self.resources({ + db: resource({ rev: 2, open: () => openSqlite(path, { wal: true }), close: (db) => db.close() }), // rev 1 → 2: reopen + port: resource({ rev: 1, exclusive: true, open: () => listen(8787), close: (s) => s.close() }), +}); +``` + +- A reload doesn't touch a resource unless its `rev` or `deps` change. +- A changed resource opens next to the old one during prepare. +- An `exclusive` resource can't exist twice, so it's swapped at the gate: close the old one, open the new one, and reopen the old one if that fails. + +### Contract changes + +| Change | What happens | +| --- | --- | +| Additive, same version | reloads; consumers keep working | +| Breaking, version bumped, with `accepts[old]` | reloads; old consumers go through the adapter | +| Breaking, without an adapter | refused, unless the consumers reload in the same batch | +| Breaking, version not bumped | refused: "consumer needs member …" | + +The host's own **eager bootstrap hold** is a lifecycle pin, not an old-version consumer. When a reload replaces +its provider, that hold adopts the new contract at commit; it does not require an `accepts` adapter for itself. +The record, cells and unchanged resources stay alive without reactivation. A refused reload keeps the original +hold and contract. Other holds (`self.get`, `ctx.get`, `host.get`, peers) still obey the compatibility rules above; +state-version bumps and migrations are still required when stored state changes. + +### Several plugins at once + +Reload them together: `host.reload([a, b])`. Prepare sees all new code at once, so a changed contract and its updated consumer check against each other. + +On the server, `host.reload(serverFacets, { remote: { "session:": workerFacets } })` reloads every connected worker too. Its report covers the server; `report.children["session:S1"]` says what each worker changed, prepared and gated. + +## Hosts and links + +```ts +// server +const server = createHost({ name: "server", principalKey }); +server.accept(webSocketLink(wss, { principal: (req) => authenticate(req) })); // stamps owner or guest from cookies +server.route("session:", { + authorize: (principal, host) => canReach(principal, host), // before resolving or spawning + resolve: (host) => workers.ensure(host), // a link to the worker, spawned if needed +}); + +// worker +const worker = createHost({ name: `session:${id}`, principalKey, upstream: ipcLink(process) }); + +// anywhere +const chat = ctx.get(Chat, input, { at: "session:S1" }); // browser → server → worker +const sessions = self.get(Sessions, {}, { at: "server" }); // worker → server, acting as worker{S1} +``` + +- **The server relays** between its children: worker to worker, and browser to worker. +- **Disconnects:** when a browser disconnects, exactly the holds it had in workers are released. +- **Principals across links:** + - a forwarded principal is accepted only from the parent link; + - calls an object makes itself act as its host; + - links ping, and a dead link releases everything held through it. +- **Reloads across hosts:** the server coordinates prepare and commit for every host. +- **Workers that start behind** (spawned after a reload) are caught up to the server's epoch before anything is routed to them. Calls their roots make to the server meanwhile run in the server's current epoch instead of failing `superseded`. +- **Browsers** don't take part in commits. They learn about new code through shape and `reset` messages, and an incompatible contract makes their calls fail with `upgrade_required`. + +## Durable + +A worker's host provides the Durable Harness as an eager local object. A Durable phase isn't a root: it's pinned to its registry snapshot by Durable itself. Each Chord call a hook or tool makes starts a fresh short root. + +```ts +host.serve(Harness, (_, _host, self) => { + const res = self.resources({ + durable: resource({ + rev: 1, + open: () => DurableHarness.open(storage, { models, registry: durableRegistry(host, Extensions) }), // a reader over the epoch tables + close: (h) => h.close(), + }), + }); + return { conversation: (id: number) => res.durable.value.conversation(id) }; +}, { eager: true }); + +// a plugin contributes a Durable extension like any registration +host.serve(Guard, (_, _host, self) => { + self.register(Extensions, { + guard: defineExtension({ + name: "guard", + hooks: [hook(ToolTask, { beforeTool: (call) => (isDangerous(call) ? { block: "Needs approval" } : undefined) })], + }), + }); + return {}; +}, { eager: true }); +``` + +A reload changing the guard: Durable phases already running keep the old snapshot, the next phase uses the new one, and there's never a phase without a guard. + +## Errors + +| Code | Meaning | +| --- | --- | +| `forbidden` | The factory or a method refused this principal | +| `invalid` | Input, arguments or a result failed validation | +| `not_ready` | A declared resource or handle was used inside a factory | +| `not_in_variant` | The member doesn't exist in the variant you hold | +| `released` | The handle was released | +| `gone` | The object was forced away while held. Call `get` again. | +| `contract_mismatch` | The provider's contract doesn't cover the consumer's, and no adapter exists | +| `upgrade_conflict` | An old root reached an object migrated meanwhile. Retry as a new root. | +| `superseded` | The root's epoch is no longer available on this host. Retry. | +| `upgrade_required` | A browser's contract is outdated. Reload its web facets. | +| `unavailable` | The target host can't be reached or started | +| `aborted` | The call's `ctx` was cancelled | + +## Rules of thumb + +| Do | Not | +| --- | --- | +| Keep reloadable state in cells | Closure variables (lost on reload) | +| Bump the state version, with `migrate`, when a cell's shape or meaning changes | Changing a schema without a bump (refused) or a meaning without a bump (silent) | +| Bump the contract version on breaking changes; add `accepts` for old consumers | Renaming or removing members at the same version (refused) | +| Do slow setup in a resource's `open` | Awaiting inside the factory (it's synchronous) | +| Do side effects in resources, listeners, tasks and registrations | Effects in the factory body (they run during prepare) | +| Use `ctx.get` inside methods | `self.get` inside methods (a new hold per call) | +| Keep roots short; long waits are state | Awaiting a human inside a method | +| Check arguments in methods | Treating the variant as all of authorization | +| Pass `ctx` everywhere | Starting background work with the caller's `ctx` (use a task) | +| Treat state values as immutable | Mutating what you read | diff --git a/packages/chord/docs/notes.md b/packages/chord/docs/notes.md new file mode 100644 index 000000000..3aa63e0b2 --- /dev/null +++ b/packages/chord/docs/notes.md @@ -0,0 +1,521 @@ +# Chord implementation notes (phase 1a) + +Running log of decisions the spec (v7) and guide did not settle, places where they contradict each other or could not +be implemented as written, and proposed simplifications. Code: `src`, tests: `test`. + +## Contradictions and gaps found + +1. **Placeholder used inside the factory.** The v7 example wired `self.listen("changes", watch.events, ...)` where + `watch` is a resource placeholder whose value only exists after activation. Resolved by the revised API: + `listener(source, select, fn)` resolves `select(resource.value)` when the record goes live. +2. **Registry syntax.** Section 8 still shows `self.register(Routes, "GET /download", handler)`; section 3 and the + guide use `self.register(Routes, { key: value })`. Implemented the object form. +3. **New records during prepare cannot always use old code.** + - Problem: "New records activated during prepare use the current epoch (old code)" fails for a type the same reload + adds. Example: reload `[clock, greeter]` where new greeter code does `self.get(Clock)`; Clock has no old code, so + the get failed with an unknown type. + - Solution: the reload pre-allocates the next epoch ("pending"). Types it adds are served from it. Their records + install behavior only for the pending epoch and are force-disposed if the reload aborts. +4. **Gate pause deadlock.** + - Problem: the gate must stop new activations of changed types, otherwise the candidate set is never final. In + phase 1a they were paused. But the second prepare pass awaits in-flight activations: if one of them declared a + hold on a new record of a paused type, it waited for the commit, and the commit waited for it. + - Resolved (accepted proposal): from the gate until commit or abort, creating a new record of a changed type fails + with `upgrade_conflict` (retryable, reason `gate`). Nothing is paused, so nothing can deadlock. The second pass + only finishes activations already in flight, bounded by the gate deadline. +5. **`upgrade_conflict` must outlive the gate.** + - Problem: the spec rejects old roots only "from the admission cut". After commit, the record still has a + behavior for the previous epoch E. A root started in E before the reload would then run old code over migrated + cells. + - Solution: gated records get `minEpoch = E'`. Calls and writes carrying an older epoch are rejected + `upgrade_conflict` permanently. This also covers replaced exclusive resources: the old code would use a closed + resource. + - Resolved further (accepted): a gated record drops its previous-epoch behavior at commit, and what only that + behavior declared is released then, not at retirement. +6. **Two-version rule vs records created after a commit.** A record of a changed type created after the commit has + only new code. A previous-epoch root calling it gets `superseded`, although the host retains that epoch. The spec + only names epochs the host does not have. Records of unchanged types share one behavior across epochs, so the issue + only affects changed types. +7. **Tasks during the gate.** + - Problem: old tasks keep running during the gate. They can write old-shaped values while `migrate` reads the cells. + They are also long-lived, so counting them as roots would block the gate until its deadline. + - Solution: tasks of gated records are aborted at the admission cut. They restart with new code at commit, or with + the old code on abort. Task contexts are code holders: handle calls made with them start fresh roots, and cell + writes with them skip gate admission (the abort covers them). +8. **Listener events during the gate.** + - Problem: each event starts a fresh root. That root has not touched the record, so a write from it would fail with + `upgrade_conflict` and the event would be lost. + - Solution: events for gated records are queued. They are replayed through the new listener at commit, or the old + one at abort. +9. **"Grandfathered activation lease" is undefined.** Implemented as: roots that called into the record and are still + running at the cut. Their re-entry is admitted, and the gate waits until they end. +10. **Remote state validation.** The spec validates state values "at the provider boundary", but per-write + validation is dev-only (revised spec). In-host consumers of remote state therefore get unvalidated values outside + dev mode. Phase 1b must decide whether links validate the snapshot when a hold is made. +11. **Local contracts.** "API is a TS type, may expose anything" conflicts with late-bound handles. A loading handle + must know whether `h.x` is a method (return a waiting caller), state (return a readable), or opaque. So members + are listed in the contract. Members missing from the contract are found only once the handle is live, by + inspecting the behavior. +12. **Input defaults and TypeScript.** Canonical identity omits fields equal to their default. TypeBox's `Static` + still makes a defaulted field required, so callers must use `Type.Optional(Type.X({ default }))` to omit it. + TypeBox has no separate input type for defaults. +13. **Adding a variant.** "Additive" per the covers rule, but an old consumer can receive a `kind` it does not know. + Implemented: provider variants ⊇ consumer variants. Members the consumer's contract does not list are hidden + (`not_in_variant` or `undefined`). +14. **Principals.** "In-host code acts as host" conflicts with roots stamped with a link principal. Implemented: a root + ctx at ingress acts as its principal (`host.root(fn, { principal })`). Contexts inside objects (method calls, + tasks, listener roots) act as `host`. `GetOptions.principal` is kept as a trusted in-host override (tests, links). +15. **Removed types.** Unspecified. Their records are force-disposed at commit instead of being kept for previous-epoch + roots. +16. **Retirement deadline.** "The next reload first waits for retirement with a deadline" does not say what happens + when the deadline is missed. Implemented: forced retirement, so those roots get `superseded` on their next call. + +## Decisions (not settled by spec/guide) + +- **Errors.** The guide's codes; a `reason` field carries the detail, for example `invalid/args`, + `invalid/cycle`, `invalid/duplicate`, `invalid/read_only`, `invalid/not_eager`, `invalid/registry_conflict`, or + `unavailable/unknown_type`. A ChordError thrown by a factory or a dependency keeps its code through every + dependent. Other errors become `unavailable`, with the original as `cause`. +- **Reload errors** (accepted): there is no `reload_aborted` code. A failed reload throws the cause's code and + reason with the message `reload aborted: …` (the cause is attached). Reasons of its own are `aborted/gate_deadline` + (with the blocking roots in `details`), `aborted/deadline`, `aborted/vote_deadline` and + `unavailable/reload_running`. Across hosts, the message names the host that failed. +- **Remote calls are validated at handle dispatch, in-host too.** A direct binding is checked against the provider's + schema, and a binding through an adapter against the consumer's. +- **Forgotten contract bumps are checked twice.** At reload, the new provider's description must cover the old + provider's when the version is unchanged, in addition to the check against live consumers and facets' `uses`. +- **Cells.** + - The tracker sees a `{ v }` box, so primitive cells work; ops are re-rooted, and `s/a/t/d` on the value itself + become `r`. + - A value returned from `change` replaces the value. It is always validated, which catches + `(d) => d.push(x)` returning a length. + - `replace` transfers ownership; values are not copied. +- **Readables.** `subscribe(fn)` delivers values, starting immediately with the current one. `events(fn)` delivers the + delta stream: `snap`, then `ops`, or `reset`. +- **Handle state members.** + - They are stable readables: `undefined` while loading or when the member is not in the variant. + - When the member moves to a different source (live, reload, variant change), subscribers get a `reset`. + - A subscription made while loading sees `snap(undefined)`, then a `reset` with the hydrated value. + - Upstream is subscribed only while someone listens. +- **Derivations.** + - Sources are record cells or derivations, recomputed atomically with the write: a dev-mode validation failure + rejects the write. Handle state members can be sources too, recomputed when they publish. + - New derivation functions are dry-run at prepare, and against migrated values at the gate. A throwing function + aborts the reload instead of breaking the commit. +- **Listeners.** A listener on a readable fires with the current value when it binds. Delivery is allowed while the + record is still finishing activation. +- **Holds.** A hold's identity is its address. A second `self.get` of the same address in one run returns the same + handle; with a different contract token it fails with `invalid/duplicate`. A changed token on reload makes a new + hold. +- **Resources.** + - New exclusive resources also open at the gate, not only replaced ones. Non-exclusive resources that depend on a + gate-opened resource open at the gate too. + - "Records depending on those resources" is read as dependents within the same record. +- **Schema change detection** compares canonical JSON, which is structural: reordering `anyOf` branches counts as a + change. +- **Disposal and re-creation.** A new record for an address that is still being disposed waits for that disposal + (exclusive resources such as ports). Abandonment is a disposal during activation: open signals abort, and resources + that opened anyway are closed. +- **Calls without ctx.** A handle call without ctx, or with a task ctx, starts a fresh root in the current epoch. + Calling a handle inside a factory throws `not_ready` synchronously. +- **Variants** change only when the factory reruns (reload). There is no dynamic `kind`. +- **Fingerprints** use a non-cryptographic 64-bit hash: it is synchronous and browser-safe, and contracts are trusted. +- **Loader.** + - Injected with `createHost({ loader: nodeLoader })`, so `index.ts` stays browser-safe because `node:module` is not + imported. + - Facets are named by module URL. + - A `module.registerHooks` resolve hook carries the cache-busting query to relative imports inside the facet + module's directory, so a plugin split into several files reloads completely. Code outside that directory, such + as Chord itself, stays a single instance. + - Old copies leak (`TODO(loader)`). +- **TypeScript.** `serve` is overloaded on the contract definition (`RemoteContract` / `LocalContract`). With a + plain `C extends AnyContract` parameter, literal `kind`s in factory returns widen to `string`. + +## Proposed simplifications (phase 1a) + +Accepted and implemented: 1 (reject instead of pause), 2 (drop the previous behavior of gated records at commit), +4 (no gating across records: only the records whose state version or exclusive resources change are gated), +5 (reload errors keep the cause's code). Open: 3, a single rule that any record whose code changed rejects +previous-epoch calls unless the change was compatible. + +The spec now uses the object form of `register` in section 8. Sections 1 and 2 say that local contracts must list +their members. + +## Phase 1b: links, relay, cross-host reload, host manager + +Files: `wire.ts` (frames), `link.ts` (memory pair, Node IPC, WebSocket), `connection.ts` (one link end on a host: +consumer, provider, relay), `remote.ts` (remote handles and readables), `cluster.ts` (coordinated reload), +`websocket.ts` (accepting a WebSocket server), `manager.ts` (host manager; Node-only, not exported from `index.ts`). + +### Contradictions and gaps found + +17. **Retirement across hosts.** + - Problem: "retire when no roots of `previous` remain" is per host, but roots span hosts. Example: a server root + R (epoch 1) calls the worker, which returns, and R waits on the server. Commit to epoch 2. The worker has no + local epoch-1 roots and retires epoch 1 at once. R's next call to the worker gets `superseded`, though the + server still runs epoch 1. + - Solution: a child retires only after its parent retired that epoch (`retire` frame, parent → child) and its + own roots of it have ended. The server decides from its own roots. + - Still open: roots started on a worker, calling up to a server that has already retired, get `superseded` + (retryable). +18. **Grandfathered roots across hosts.** + - Problem: the gate grandfathers "roots that touched the record and are still running". On the worker, a server + root is visible only while one of its calls runs there. In the example above, the worker forgot R's lease when + the call returned. The gate then did not wait for R, and R's re-entry got `upgrade_conflict` instead of being + admitted. + - Solution: a `root_end` frame. Every host remembers on which links a root's calls went out; when the root ends + (at its origin, or on receiving `root_end`), it sends `root_end` on those links. A root that came over a link + keeps its leases until then, or until that link closes. Leases outlive calls, so a root whose origin never + ends it (a bug) blocks gates until their deadline. +19. **`get` frame.** The spec lists `get`/`hold`/`release`. One acquisition frame is enough: `hold` creates the remote + handle and returns `shape` or `gone`. There is no separate `get`. +20. **State subscriptions** need `sub`/`unsub` frames, which the spec does not list. Remote state members hydrate only + while subscribed: `h.view.value` is `undefined` until someone subscribes. The guide's "undefined until hydrated" + holds, but hydration requires a subscriber. Auto-subscribing on `.value` would leak subscriptions. +21. **Commits must not throw.** + - Problem: committing on the worker pushed a shape to a browser hold whose contract no longer bound. Building the + shape re-bound the hold, threw `contract_mismatch`, and the commit stopped halfway. + - Solution: shape and view access never throw (bind failures are reported), and browser holds are checked + first. Lesson: everything a commit does to other parties (shape pushes, resets, registry notifications) must + be failure-isolated. +22. **Epoch ids are global.** The server issues them. (Superseded by the Astra fixes below, #5.) Children no longer + adopt or renumber epochs. A child that connects behind the server is fenced and reconciled. +23. **"Server persists the decision"** (section 7.5) is not implemented: no storage exists in Chord. A server crash + between its own commit and a child's commit leaves that child on the old epoch until it reconnects; same remedy + as 22. +24. **Only one level of reload participants.** The coordinator's direct child hosts take part. A child does not + forward `prepare` to its own children (the server, workers, browsers topology has none). +25. **Browser holds in reloads.** + - Problem: a browser hold whose contract no longer binds would refuse the worker's reload, through the hold the + server relays on the browser's behalf. + - Solution: the relay marks such holds `client: true`. Providers skip them in the reload's contract check and, + at commit, make them `gone` with `upgrade_required`. A browser holding an outdated token on a new `hold` also + gets `upgrade_required` instead of `contract_mismatch`. +26. **Links that drop.** + - An upstream link may be replaced (`host.connect`): remote handles go back to `loading`, re-hold over the new + link, and readables resubscribe. Subscribers get a reset. + - A downstream host that disconnects makes the remote handles held through it `gone` (`unavailable`). + - The provider releases everything held through a closed link at once. The re-held record is therefore a new + one: the counter in the reconnect test restarts at 0. Keeping holds for a reconnect window would need session + ids on links; not done. + - Calls in flight when a link drops reject `unavailable`; it is unknown whether they ran. +27. **Refused clients.** The WebSocket adapter closes unauthenticated sockets. On the client this looks like a + dropped upstream: calls wait for a reconnect until their ctx is aborted. A refusal reason in `bye` could tell + clients to stop retrying. + +### Decisions + +- **Principals across links.** + - A downstream link acts as the principal it was accepted with, whatever a frame says. + - Frames from the parent may forward a principal (relay). Without one, they act as `{ host: }`. + - Clients never carry roots or epochs: the server starts a root for each of their calls. +- **Relay authorization.** Relaying needs a matching `route`, because the route holds `authorize`. + `authorize(principal, hostName)` runs before `resolve` and again after. A host's own gets may use a directly + connected child by name without a route. +- **Relay mechanics.** + - The relay works on frames: it rewrites ids and passes delta-encoded state frames through untouched, keeping one + encoder/decoder pair per origin subscription. + - Calls and subscriptions sent before the route resolves are queued. + - A relayed call runs inside a server root (it joins the caller's root, or starts one for a client), so it counts + for the server's epoch bookkeeping. +- **Snapshot validation.** A provider validates each snapshot and reset of a remote state member against its + contract schema, once per (re)subscription. Ops are validated only in dev mode. This settles 10 for links. +- **`invalidatePrincipal(match)`.** + - `match` is a predicate (this host only) or a pattern object, where listed fields must be equal. Patterns + propagate to child hosts in an `invalidate` frame. + - Matching links are re-stamped (`accept({ restamp })`) or closed. Re-stamping keeps a browser connected under its + new grant revision; its old relayed records are force-disposed, so its handles become `gone`. +- **Liveness.** Each connection pings every `pingInterval` and closes the link after `pingTimeout` without any frame. + Closing releases everything held through the link. +- **Contract tokens over links.** A hold sends the consumer's description. The provider rebuilds a token from it + (`contractFromDescription`) and binds it with the covers rule. +- **Cross-host reload.** + - `host.reload(plugins, { remote: { "session:": [modules] } })` on a server with child hosts coordinates + automatically. Every connected child host prepares, even with no modules, so epoch ids stay in step. + - Votes have a deadline. Any failure aborts every host, and the error names the host. + - A child cannot reload on its own (`unavailable/not_coordinator`). +- **Host manager.** + - `forkDriver({ entry, execArgv, env, init, lock })` forks a detached child, so the child leads a process group + and the tree can be killed. + - The lock file holds the child's pid and is taken over when that pid is dead. + - Control messages share the IPC channel with frames: `{ chord2control }` next to `{ chord2 }` (the wire keys keep their chord2 names). + - `stop` sends `shutdown`, waits `grace`, then kills the process group. Start, stop and replace are serialized, + and replace waits for the exit before spawning. + - Liveness (`activeCalls`, `viewers`, `durableTasks`) is reported by the child with `childControl().report`. The + manager derives `booting` and `idleFor` and stops the child when `keepAlive(liveness)` returns false. +- **Chord stays dependency-free.** WebSocket support is structural: browser WebSocket, Node's global WebSocket and + `ws` sockets all fit `WebSocketLike`, and `acceptWebSockets` takes any server with `on("connection")`. Only the + tests import `ws`. + +### Left open + +- The Durable registry bridge (`RegistryReader` per epoch) is not implemented. +- Reconnect windows (session ids), restarting a stale child after a missed reload (22, 23), multi-level reload + participants (24), and a refusal reason for clients (27). + +## Review fixes (Sol, phase 1a findings #1-#11) + +Regression tests: `test/review.test.ts` (one or more per finding, plus ctx races), +`state.test.ts` (#3), `cluster.test.ts` (abort while a worker prepares). Each was checked to fail with its fix +reverted. #5 has two layers (no disposal while invocations run; disposal awaits invocations), and its test fails once +the disposal-side layer is reverted. + +1. **Partial epochs.** + - Problem: the commit published the epoch, then ran fallible installs. Example: three records; the second's new + listener source throws on subscribe. The reload rejected, but the epoch had advanced and the records mixed old + and new code. + - Fix: everything fallible runs at the end of prepare. + - Listener subscriptions are staged: subscribed with buffered delivery, activated or discarded later. + - A listener whose source did not change keeps its subscription; only its function is swapped (by name), so + events are neither duplicated nor lost. + - Migrated values and derivations are dry-run and validated against the prospective schemas. + - The commit then only installs, inside `holdPublications`: no observer runs until the whole epoch is in place. + - If an install still throws, the host fails terminally: `host.failed` is set and every admission fails + `unavailable/host_failed`. The host manager should restart such a host. Children that already voted cannot be + rolled back after the server's commit, so terminal failure is the only safe answer there too. +2. **Obsolete schemas during migration.** + - Problem: `cell.reset()` validated the migrated value (dev mode) against the remote member schemas of the old + behavior, so a `{n}` → `{total}` migration failed during the commit. + - Fix: the dry run validates every migrated value and derivation against the cell's new schema and the new + behavior's member schemas. The commit installs the schemas (`refreshSchemas`), then the values, without + re-validating. +3. **Re-entrant publication.** + - Problem: subscribers were notified synchronously, before derived values were installed. Example: a re-entrant + writer reacting to `n = 1` by writing `n = 2`; a second subscriber saw `[0, 2, 1]` and `n * 2` ended at 2. + - Fix: one process-wide FIFO queue (`publish.ts`). A publication adopts the cell and every derived revision, then + enqueues notifications, each captured with the subscribers of that moment. Shape pushes, registry + notifications, handle readables and listener invocations go through the same queue. + - Behavior change: a listener invocation from inside another delivery now runs after the current delivery instead + of nested in it. Outside a delivery it still runs synchronously. +4. **Listener roots and leases.** + - Problem: listener roots never took a lease on their record, so a migration could commit while an async listener + was still running; the old callback then read migrated cells. + - Fix: a listener invocation takes a lease on its record (the gate grandfathers it and waits). It runs with a + child ctx cancelled by forced disposal, and counts as an invocation. +5. **Resources closed under running methods.** + - Fix: methods and listener invocations keep the record alive until they settle. A record with invocations is not + disposed (`#maybeIdle`). Disposal (released or forced) stops the runtime, awaits invocations, then closes + resources. Forced disposal first makes handles `gone` (no new calls) and cancels running invocations. +6. **Activation finalization outside the failure path.** + - Problem: finalization ran in the success handler of `.then(success, failure)`. A registry conflict between + concurrent eager records rejected `ready` but left the loser live with its resource open. + - Fix: finalization runs inside the activation transaction: registry check, behavior install, runtime start and + live status in one synchronous step. Any failure aborts the record's invocations and stops its runtime, + unregisters, awaits invocations and unwinds. + - A release during listener startup aborts the activation (no resurrection). +7. **Prepare touching retained resources.** + - Fix: `self.resources` returns facades. `.value` throws `not_ready` while any factory runs, even if the resource + is open. + - Records served only by a reload's pending epoch (types the reload adds) defer listeners, tasks and registrations + to the commit (`deferredRuntime`). Their listeners are staged with the rest. +8. **Same-batch breaking change.** + - Fix: holds declared by a prepare rerun bind and normalize against the pending service (`candidate`), so a + breaking provider change and its updated consumer reload together. +9. **Exclusive-resource rollback.** + - Fix: a replaced exclusive resource is recorded for the rollback before its close runs. The rollback reopens it, + and a resource that cannot be reopened is marked `unavailable`: its `.value` throws `unavailable` instead of + looking merely not open. +10. **Retirement order.** Fix: retirement closes dropped resources (reverse open order) before releasing dropped + holds, the same order as disposal. +11. **`covers` false positives.** + - Problem 1: `fits` ignored keywords it did not model. + - Fix: a provider schema with any validation keyword outside the modelled set only fits an identical schema. + Unmodelled keywords include `not`, `allOf`, `contains`, `$ref` and `additionalProperties` given as a schema. + - `patternProperties` and consumer `additionalProperties` schemas are now checked against provider properties. + - Problem 2: the additive rule ("extra fields fine") is only sound if a consumer never sends a field its own + schema does not know. + - Fix: arguments are stripped (`Value.Clean`) with the consumer's schema, then with the provider's, before + validation. Remote handles strip before sending. + - Counterexample tests check each case with `Value.Check`: a value valid under the consumer's schema and invalid + under the provider's must make `fits` false. + +### Races fixed + +- **Aborted task contexts.** Calls made with a task ctx after the task was aborted started fresh roots without its + signal. Now the call is refused (`aborted`), and the fresh root carries the task's signal. +- **Expired and foreign contexts.** + - A ctx whose call or root already settled is refused (`aborted/expired`). + - A ctx from another host is refused (`invalid/foreign_ctx`). + - Both apply to calls and cell writes. `ctx.get` after settlement now fails `aborted/expired` (was `released`). +- **Delayed opens.** A resource open that completes after its signal aborted (rollback, abandonment) closes its + value at once and is not recorded as open. +- **Abort while a worker is still preparing.** The child used to ignore an `abort` that arrived before it voted, + keeping its reload slot forever. Now an abort during loading or prepare rolls the prepare back, frees the slot and + sends no vote. An abort that arrives before the prepare even started is remembered. +- **Observers during a commit.** An observer of record A that calls record B during a commit now always sees B's new + code. + +### Still open + +- The publication queue is process-wide and synchronous. A slow observer delays every later notification in the + process; per-host queues would isolate hosts in one process (tests, in-process workers). +- Terminal host failure has no automatic restart inside Chord. The host manager's keepAlive or `fatal` path must + pick it up. + +## Review fixes (Astra, phase 1b findings #1-#13) + +Regression tests: `test/links-review.test.ts` (one or more per finding, plus two browsers sharing one worker +connection), `manager.test.ts` (#8, #12, #13), `websocket.test.ts` (#1). Each link test was checked to fail with its +fix reverted. #2's generation check is backed by `revoke()` closing pending relays at once, so its test fails only +when that is reverted. + +1. **Self-declared peers.** + - Problem: a peer's `hello` decided its kind and name, and with them root honoring, reload participation and + routing identity. + - Fix: the acceptor decides. `accept(link, { principal, kind, name })`: `kind` defaults to `client`, and a `host` + needs its `name`. A hello that disagrees with the configuration closes the link. + - Only configured host links are addressable by name, carry roots and may vote. Control frames (`prepare`, + `commit`, `abort`, `retire`, `invalidate`) are accepted only from the parent, and `vote` only from a host child. + Anything else closes the link. + - `acceptWebSockets` always accepts as `client`. + - `root_end` is honored only from a link the root actually came over (foreign roots record a set of links), and a + root ends when no such link remains. +2. **Revocation racing route resolution.** + - Problem: revoking or re-stamping a principal while its relay was still resolving the route did not stop it; the + relay then forwarded the old principal and created a worker record for it. + - Fix: each origin link has an authorization generation. Revoking or re-stamping bumps it and closes the link's + relays at once, including those still resolving. + - `HostImpl.relay` re-checks the generation after every await of authorize/resolve/sync and right before + forwarding. +3. **Child prepare cancellation.** + - Problem: the child registered the reload only after its prepare finished, so an abort or disconnect arriving + earlier was forgotten and the child stayed prepared, gated and reloading forever. + - Fix: `ChildReloads` registers the operation (`loading` → `preparing` → `prepared`, plus `cancelled`) when the + `prepare` frame arrives, before any await. An abort or a lost parent marks it cancelled and rolls back what + exists. The operation unwinds when its pending work settles, without a vote, freeing the reload slot. + - A second `prepare` while one runs gets a refusing vote. +4. **FIFO commit ordering.** + - Problem: the server's commit started tasks and ran listener and registry callbacks before the child commit + frames were sent, so new-epoch calls could reach a child before its commit (`superseded`). + - Fix: `Reload.commit(beforeStart)` installs everything, then calls `beforeStart`, where the coordinator queues + the commit frames. Only then does it start tasks and eager activations and release the held notifications + (listener invocations, registry and state callbacks). Anything the new epoch sends to a child follows its commit + frame on the FIFO link. +5. **Relabeled old code.** + - Problem: a child that voted, then lost the link and rolled back, was renumbered to the server's epoch on + reconnect; its old code then ran under the new epoch id. + - Fix: `adoptEpoch` is gone. A host child's hello carries its epoch. + - In step: usable (`synced`). + - Behind: fenced (direct gets, routes and relays wait for `whenSynced`) and reconciled. The server runs the + cluster protocol against that child alone, with the modules cluster reloads gave its name prefix, to the + server's current epoch; it retries if the server moved meanwhile. + - Ahead: the link closes. + - New workers spawned after a reload are reconciled the same way, which costs one reload round on their first + connect. Fenced children are not participants of other reloads. + - The child's epoch is never renumbered: reconciliation either loads the expected code or the link closes. +6. **Leaked leases on round trips.** + - Problem: a server root that went to a worker and came back was marked foreign on the server, never ended, and + its lease blocked later gates until their deadline. + - Fix: roots started on a host are tracked as local (`#local`) and never become foreign there, even when they come + back over a link. + - Root ids carry the host's incarnation (`name#incarnation:n`), so a restarted host with the same name never + reuses an id. +7. **Disconnects not cancelling downstream.** Fix: relays own their calls (`RelayCall`). A cancel, an origin + disconnect, a release, a revocation or the downstream going away settles each call once, forwards `cancel` when + the downstream hold still exists, and removes both id mappings. +8. **PID-file lock.** + - Problem: the lock was a PID file taken by the manager. Two managers could both read the same stale PID and take + over, and the child never held the lock itself. + - Fix: the child holds a kernel lock: an abstract socket on Linux/Android, a named pipe on Windows, a probed + socket file elsewhere (the approach Pi Mobile uses). `childControl().lock()` acquires it before storage opens. + - The manager passes the lock name in the environment. A second child fails `ready` with + `unavailable/locked`, which the manager reports as the start error. +9. **Relay release and unsubscribe races.** + - Fix: `RelayHold` owns queued and forwarded subscriptions and calls. An unsubscribe before resolution drops the + queued entry, so nothing is forwarded later. + - A release or close during resolution rejects waiting calls and forwards nothing. Any close removes every + mapping on both links. +10. **Malformed frames and duplicate ids.** Fix: every received frame is validated against a TypeBox schema + (`isFrame`). Malformed frames, duplicate hold/call/sub ids, and control frames from the wrong side close the link. +11. **Snapshots through `accepts` adapters.** + - Problem: snapshots through an `accepts` adapter were validated against the provider's schema instead of the + consumer's. + - Fix: they are validated against the consumer's contract, because the adapter's view is what the consumer gets. + Direct bindings still use the provider's schema. + - Handle readables also accept any readable an adapter returns, not only cells and derivations. +12. **Descendants surviving a graceful exit.** Fix: the process group is killed when the leader exits, on every + path (graceful, crash, kill). +13. **Idle clock.** Fix: `idleFor` counts from the busy → idle transition (or from ready), not from the last busy + report. + +### Remaining limits + +- Reconciliation needs the child's module list. It is remembered from cluster reloads (`host.remoteModules`), so a + server restart forgets it, and a child behind a restarted server then reloads no modules: it gets the epoch id with + its own code. Persisting the module list with the epoch would close this. +- The vote deadline for reconciliation is fixed at 30 s. A failed reconciliation closes the link, so the manager + restarts the worker. +- On non-Linux Unix, two processes that probe the same stale socket file at the same moment could both take it + (accepted for Pi Mobile too). Managers serialize starts per lock name. + + +## p2 requests (docs/requests.md #1-#7) + +Tests: `test/requests.test.ts`; each was checked to fail with its fix reverted (#1 also at the type level). + +1. **Held state members as behavior state.** Behavior state members are typed `StateSource` (`Readable | + Readable`), and `validateBehavior` accepts any readable, not only cells and derivations. Such a member + is not validated per write in dev mode (it is not this record's source); snapshots over links still are. +2. **Facet host access.** `FacetApi` is `{ serve, name, root, registry }`, for `install`, `load` and `reload` alike. + Not added to `self`: methods have `ctx`, and a factory should not start roots. +3. **Listener resubscription.** A subscription is kept when the listener's origin (the resource, not its facade, or + the source) and its `select` are unchanged; `select` counts as unchanged if identical or with the same source text. + A kept listener's `select` is not called again. Limitation: a `select` whose behavior depends on captured values + that change between factory runs is not detected (documented in the guide). +4. **`replace` diffing.** `replace(ctx, value, { diff? })`. Default: diff when the new value reuses at least one + container of the old at the same key (a new revision of the same state), else `r`. The tracker still adopts a whole + replacement; only the published ops come from `diffRevisions`. Unrelated replaces still publish `r` (existing + behavior). +5. **Roots of a fenced child.** The parent records the epoch a child announced while behind (`fencedEpochs`). A call + from that link carrying a root of such an epoch is admitted in the parent's current epoch at its first call and keeps + that epoch for the root's later calls (`superseded` once the parent retires it). Other unknown epochs stay + `superseded`. Notes item 17 (a worker root calling a server that retired an epoch both had) remains open. +6. **`self.expose(resource)`.** Returns an `Exposed` marker accepted where a local `opaque` member is expected. + Handles resolve it to the resource's value when live (`undefined` while loading); the resource must be declared by + the same record. +7. **Per-child reload reports.** A child's vote carries its prepared report (`Reload.summary()`); the coordinator's + `ReloadReport.children` maps host names to them. Additive; local reloads have no `children`. + +## p2 requests #8-#10 + +Tests: `test/requests.test.ts` (#8-#10); each fails with its fix reverted. + +8. **Stale remote state members.** A `RemoteReadable` forgets its value when its last listener leaves (it is only + kept current while subscribed). A new subscriber gets `undefined`, then the snapshot as a reset. Local handle members + always read the current source and were not affected. +9. **`handle.onStatus(listener)`** on local and remote handles: called (through the publication queue) after every + status change, with the error for `gone`. Remote handles also report `live` → `loading` while an upstream link + reconnects. `onStatus` is now a reserved member name. +10. **Shutdown order.** `host.close()` and `invalidatePrincipal` dispose in waves: a record goes once no remaining + record holds it, so holders close their resources while their holds (local and remote) still work. `host.close()` + closes links only after the records. Caveat: a `close` that awaits a remote call over a link that is already down + waits for a reconnect (calls without a signal never time out); the guide says to bound such calls. + +## p2 request #11: eager startup readiness + +Tests: `test/readiness.test.ts` (load/install activation delays, factory/dependency/resource failures, +missing providers, same-batch provider ordering, lazy preservation, configured eager input, readiness snapshots, +and deferred worker admission). + +- **Problem:** load resolved after setup, while eager activation failed only through `onError`. Startup could claim + readiness with a required object unavailable. Waiting for everything before worker control readiness would also + deadlock: reporter waits for server SessionStatus; parent waits for worker readiness before accepting its link. +- **Solution:** install returns an eager-activation promise; load installs its entire batch synchronously before + awaiting those promises. Failures use the existing handle activation errors, and still reach `onError`. Lazy + objects remain lazy unless held by an eager dependency. Setup is still synchronous; installation is not rolled + back by activation failure. +- **Bootstrap escape:** additive `LoadOptions.awaitEager` (default true) supports false on load/install. It starts + eager activation without awaiting it. `host.ready()` later awaits the eager holds present at call time, including + direct serves. No upstream-dependency detection or extra holds are needed. Connect the worker upstream before + loading, await the independent local Harness handle, announce control readiness, then await the full barrier. + Application control admission, bootstrap liveness and fatal reporting remain application responsibilities. +- **What p2 can drop:** eager declaration collection around facet `serve`, synthetic eager handles kept solely for + startup, and explicit per-eager `ready` loops in the loader/testHost. Ordinary startup uses `await host.load`. + Worker startup replaces its upstream eager-ready loop with `await host.ready()` after admission; retain the local + Harness readiness wait, parent control announcement, and busy/fatal bookkeeping. This task does not change p2's + loader or worker integration. + +## Owner decision: keep the state API + +The proposal to give each object exactly one state value and drop `cell()` and `derivation()` is rejected. The state +API stays as it is: named cells and derivations per object. Do not reopen this without a new, concrete problem. diff --git a/packages/chord/docs/requests.md b/packages/chord/docs/requests.md new file mode 100644 index 000000000..631d93578 --- /dev/null +++ b/packages/chord/docs/requests.md @@ -0,0 +1,122 @@ +# Chord requests (from p2) + +p2 is the first application built on Chord. These are the bugs, gaps and missing features found while building it. Each has the workaround p2 uses meanwhile. +#1-#7 are fixed in Chord (see notes.md, "p2 requests"); p2 dropped their workarounds. +Fixes live in `src`; additional resolved requests are marked below. + +## 1. Handle state members cannot be returned as behavior state members (typing) + +- Problem: the guide shows `return { kind: "viewer", view: c.view }` (a held object's state member exposed as this + object's state). The handle member is typed `Readable`, the behavior member `Readable`, so this + does not type-check. +- Workaround: `self.derive({ view: derivation([c.view], (v) => v ?? null) })`. Costs one more diff per change. +- Request: accept handle readables in `BehaviorOf` (they are `undefined` only while loading, and the record is live + before its behavior is used), or a helper that narrows them. + +## 2. Facets have no host access outside `serve` + +- Problem: `FacetApi` only has `serve`. Code that is not a Chord method, listener or task needs a host to start a + root or read a registry: Durable tools and hooks (`rename_session` calls the server), a method reading another + registry (`HarnessConfig` in `Chat.configure`). +- Workaround: the worker entry stores its host in a runtime module (`workerHost()` in `src/runtime/worker.ts`). +- Request: `FacetApi.root(fn)` and `FacetApi.registry(r)` (or `self.root`/`self.registry` on records), so plugins do + not reach for a process-global host. + +## 3. A listener's `select` must return the same object for the same value + +- Problem: at reload, `stageListeners` keeps a subscription only when `select(value)` returns the identical + subscribable. A `select` that wraps the value in a new adapter object (natural for Durable watches) resubscribes + at every reload. For a source that accepts one subscriber (a Durable `WatchHandle.start`), the reload aborts: + `reload: Watch is already started`. +- Workaround: `watchSource()` and `summaries()` memoize their adapter per underlying value (WeakMap). +- Request: document it in the guide (`listener(source, select, fn)`), or compare `(resource, select)` instead of the + selected object (a changed `select` function would still resubscribe). + +## 4. `cell.replace` publishes the whole value + +- Problem: mirroring an external immutable state (a Durable conversation view, revised on every streamed token) + into a cell with `replace` sends a full `r` op to every remote subscriber per revision. `change` only diffs what a + draft mutates; there is no way to hand over a new immutable revision and get a diff. +- Workaround: the cell holds the revision; a derivation `(v) => v` exposes it, and `Derived.install` diffs + revisions with `diffRevisions` (cheap, unchanged subtrees are shared). +- Request: `cell.replace(ctx, value, { diff: true })` (or diff by default when the old and new values share + structure). + +## 5. Roots of a child behind the server get `superseded` + +- Problem: a worker spawned after a cluster reload starts on epoch 1 and is reconciled to the server's epoch. Its + eager objects run meanwhile; a listener root of epoch 1 calling the server gets `superseded: epoch 1 is not + retained by server`. The event is lost unless the caller retries. Notes item 17 lists the general case as open. +- Workaround: `retrying(host, ctx, fn)` (`src/runtime/retry.ts`) retries retryable errors in fresh roots; the status + reporter re-reads the latest task graph on each attempt. +- Request: hold upstream calls of a fenced child until it is reconciled, or have the parent admit such roots in its + current epoch (the child had no old roots there). + +## 6. Exposing a native object opened by a resource + +- Problem: local members must exist when the factory returns, but a resource's value only exists after activation. + An `opaque` member therefore cannot hand out, say, the Durable Harness opened by a resource. +- Workaround: methods returning it (`harness.harness(null)`), which makes every use asynchronous. +- Request: an `opaque` member backed by a resource (`self.expose(res.durable)`), readable once the record is live. + +## 7. Cluster reload report does not include children + +- Problem: `host.reload([], { remote })` on the server returns the server's `ReloadReport`; `changed` and `prepared` + are empty when only worker facets changed, so a caller cannot tell what the workers did. +- Request: per-child reports in the coordinator's result. + +## 8. An unsubscribed handle member keeps a stale value + +- Problem: a remote handle's state member subscribes upstream only while someone listens. After the last listener + leaves, `value` keeps the last received value, and the next `subscribe` first delivers that stale value + synchronously, before the fresh snapshot arrives as a reset. Example (p2's sharing test): `until(drafts.drafts, + (v) => v.length > 0)` resolved at once with a draft that had been claimed after the previous `until` unsubscribed; + `poll(() => sharing.guests.value?.length === 0)` never saw the guests leave. +- Workaround: tests keep a subscription open while they wait for changes (`until`, or an explicit `subscribe`); + React's `useReadable` subscribes for the component's lifetime. +- Request: reset `value` to `undefined` when the upstream subscription ends (a new subscriber then sees `undefined`, + then the snapshot), or document it in the guide (State, Handles). + +## 9. No public status event on handles + +- Problem: a UI holding a handle must notice `gone` (revocation, worker stopped) to `get` again. `onShape` fires on + live/variant/reload, not on `gone`; `goneListeners` is internal. +- Workaround: `useObject` (src/runtime/web.tsx) checks `handle.status` every second after `ready`. +- Request: `handle.onStatus(listener)` (or `onGone`). + +## 10. Host shutdown disposes held objects before their holders + +- Problem: when a worker host closes, a `Chat` record's resource `close` read `undefined` from the `opaque` member of + its held `WorkerProcess` handle (`TypeError: Cannot read properties of undefined (reading 'add')`). The guide says + an object's own holds are released after its resources close, so a dependency should still be live there. +- Workaround: the resource captures what it needs at `open` and uses that in `close` (p2's chat worker). +- Request: at host close, dispose holders before what they hold (reverse dependency order), or document that + handles may be gone in `close` during shutdown. + +## 11. Loading facets does not await eager activation failures + +- Problem: `await host.load([facet])` can resolve even when an eager factory fails immediately (for example, + `self.get(ModelConfig)` with no models plugin). The failure only reaches `onError`; startup/tests can report ready + while a required object is already unavailable. +- Workaround: p2's loader and `testHost` collect `serve(..., { eager })` declarations and explicitly await their + handles' `ready`. The worker opens its Harness, announces control ready (the parent then accepts its IPC link), + and only then awaits upstream eager holds, counting this bootstrap as an active call. Waiting for a status + reporter before announcing ready deadlocks: + reporter waits for server SessionStatus, server waits for worker ready before accepting that link. +- Request: an awaited activation-ready API for loaded eager objects, with startup errors propagated, and a documented + separation between local bootstrap readiness and upstream admission. Keep ordinary lazy objects lazy. + +## 12. An eager object's bootstrap hold pins its old contract (resolved) + +- Problem: chat-mode changed its store from one boolean to per-conversation state, with a contract-version bump + and state migration. When the store itself was eager, its host-owned hold still required version 1, so reload + failed with `contract_mismatch` even after every real consumer was updated. +- Historical workaround: a separate eager router held a non-eager store. This separation is no longer required + for contract upgrades (it may still be useful organization). +- Resolved: `src/reload.ts` excludes only the exact eager bootstrap hold from old-consumer compatibility + checks, then replaces it with a hold on the new provider contract at commit. The replacement attaches before + the old hold releases: the record, migrated cells, unchanged resources and registrations survive. Abort leaves + the old bootstrap hold untouched. Other consumers keep their own tokens and still need coverage/`accepts` or + replacement in the same batch; forgotten version bumps and missing/invalid state migrations remain refused. +- Tests: `test/eager-reload.test.ts` covers configured input, consecutive contract bumps, migration/rollback, + resource lifetime, external and record-owned consumers, adapters, forgotten bumps and cluster vote failure. diff --git a/packages/chord/docs/spec.md b/packages/chord/docs/spec.md new file mode 100644 index 000000000..93a1817b6 --- /dev/null +++ b/packages/chord/docs/spec.md @@ -0,0 +1,225 @@ +# Chord specification (v7) + +Status: implemented in `packages/chord`; this document was its design target. Synthesized from the round-6 reviews and the hot-reload designs of Sol, Astra, Opus and the DeepSeek-harness agent. Supersedes the `chord-guide` artifact v1 and design v6 where they disagree. + +Hard constraints: +- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins plus `typebox` for schemas. No other Pi packages. +- Built-in plugins are trusted, loaded from source in-process. The plugin loader uses dynamic `import()` with a cache-busting query (owner-approved exception, `TODO(loader)`: replace with an isolating, evicting loader; old module copies leak; TUI plugins need isolation). +- Hot reload is required. Host restart only for Chord/runtime code changes or host-level crash recovery. + +## 1. Concepts + +| Term | Meaning | +| --- | --- | +| type | Name + contract. `remote` (addressable from other hosts; schemas for input, variant state members, method args/results) or `local` (host-only; members may be anything, including functions and native objects, but must be listed in the contract: `method`, `opaque` or a state schema). | +| factory | `(input, principal, self) => behavior`. Synchronous. Registered with `host.serve(type, factory, options)` from a plugin facet. | +| record | Stable identity per `(type, canonicalInput, principalKey)`. Owns cells, resources, holds, listeners, tasks, registrations. Survives reloads. | +| behavior | What a factory run returns (methods + state members; `kind` for remote variants). Immutable, per epoch. | +| handle | What `get` returns. Late-bound. Never the record or behavior. | +| hold | One handle's claim. A record lives while held. | +| principal | App-defined. Stamped by links. In-host code acts as the `host` principal. | +| root | One short Chord call tree, admitted at trusted ingress (link request without root, UI event, timer, `host.root`). Carries `{ rootId, epoch }`. | +| epoch | One committed set of behaviors/registrations per host (server-issued id). | +| code holder | Long-lived execution that is NOT a root: Durable phases/tool runs, browsers, `self.task` loops. Every Chord call they make starts a fresh short root. | + +## 2. Contracts + +Contracts are runtime values for every type (shipped bundles have no TypeScript; third-party plugins are built separately). TS types are inferred from them. + +```ts +export const Chat = remote("pi.chat", { + version: 1, // contract version: a plain number, bumped only on breaking changes + input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + variants: { + viewer: { state: { view: ViewSchema } }, + participant: { + state: { view: ViewSchema }, + methods: { send: method(SendSchema, Type.Object({ submissionId: Type.Number() })) }, + }, + }, +}); + +export const Index = local("pi.index", { + version: 1, + input: Type.Object({ session: Type.String() }), + members: { + search: method(Type.Object({ query: Type.String() }), Type.Array(Hit)), + onChange: method(Type.Function([Change], Type.Void()), Unsubscribe), // non-JSON allowed for locals + harness: opaque("pi.durable-harness@1"), // native object, by name+version + }, +}); +``` + +- Input identity: normalize (apply defaults, strip unknown), validate, then canonical JSON with fields equal to their schema default omitted. Changing defaults or normalization requires a new type name. +- Remote calls validate args/results and state values at the provider boundary. Local calls are not validated per call. +- Local contracts list every member (`members: { search: method(...), harness: opaque(...), status: Schema }`). A handle that is still `loading` must know whether `h.x` is a method (a caller that waits until live), a state member (a readable) or an opaque value, before any behavior exists. Members missing from the contract are found only once the handle is live. +- Every bundle embeds the contract tokens it was built against (its own copy). Fingerprint = hash of the canonical description. + +### Compatibility rule + +A consumer built against version M binds to a provider at version N iff N = M and the provider's description covers the consumer's; otherwise only through `accepts[M]` adapters declared by the provider (`serve(type, factory, { accepts: { 1: (current) => v1View } })`); else `contract_mismatch`. + +Covers: every consumer member exists with the same kind; methods: provider accepts every argument the consumer can send (new fields optional, nothing newly required or retyped) and the provider's result fits the consumer's expected result (extra fields fine); state: provider value fits consumer schema (extra fields fine); opaque: same name@version. + +Additive changes need no bump. Breaking changes (remove/rename/retype, newly required, or a change of meaning with the same shape) bump the number. Same version + non-covering description = forgotten bump, refused. Meaning changes without a bump are undetectable (author's responsibility, as with Durable document versions). + +Checked at: `get` binding, reload prepare (new provider token vs every surviving consumer holding the type and every loaded plugin importing it), remote holds (consumer token in the hold frame). + +The exact host-owned eager bootstrap hold is a lifecycle pin, not a surviving old-contract consumer. When its +provider is replaced, commit replaces that hold with one bound to the new provider contract, acquiring it before +releasing the old hold so the same record, state and unchanged resources survive. It needs no `accepts` adapter +for its own previous contract. Prepare/abort do not change that hold. All other consumers retain their own tokens +and the normal coverage/adapter checks; state-version and migration checks are unchanged. + +## 3. Factory and declarations + +Factories are synchronous; returning a promise throws. Declarations are identified across factory reruns by **property names** in object literals (not call order, not separate string keys). Names are unique per record per declaration kind; a declaration method may be called more than once if names don't repeat. Holds are identified by their address. + +```ts +host.serve(StatusStore, ({ session }, _host, self) => { + const state = self.state({ + status: cell(StatusSchema, { running: false, tasks: 0 }), // schema required, initial value + errors: cell(Type.Array(Type.String()), []), + }); + const catalog = self.get(Catalog, {}); // hold; identity = canonical address (+ at) + const res = self.resources({ + watch: resource({ + rev: 1, deps: [catalog], exclusive: false, + open: async ({ signal }) => openWatch(session, signal), + close: async (w) => w.close(), + }), + }); + self.listen({ + changes: listener(res.watch, (w) => w.events, (event, ctx) => state.status.change(ctx, (d) => { d.tasks = event.tasks; })), + }); + self.tasks({ poll: async (ctx) => { /* loop until ctx.signal aborts */ } }); + const derived = self.derive({ visible: derivation([state.status], (s) => project(s)) }); // synchronous + self.register(Routes, { "GET /status": handler }); // registry keys are the registry's own keys; eager records only + return { status: derived.visible, report: (args, ctx) => { state.status.replace(ctx, args); return null; } }; +}, { + state: { version: 2, migrate: { status: (old, from) => upgrade(old, from) } }, // migrate gets the old value and its version + linger: 30_000, + eager: false, +}); +``` + +| Declaration | Survives reload | Notes | +| --- | --- | --- | +| `state({ name: cell(schema, initial) })` | same cell and value; migrated on state-version bump | Schema required (no TypeScript at runtime). `change(ctx, draft => void)`, `replace(ctx, value)`, `value`, `subscribe`. Writes through `change` touch only mutated paths. | +| `get(type, input, { at? })` | kept if declared again (same address) | returns a handle | +| `resources({ name: resource({ rev, deps?, exclusive?, open, close }) })` | kept unless `rev`/`deps` change | `open` runs at prepare (non-exclusive) or at the gate (exclusive); handle exposes `.value` | +| `listen({ name: listener(source, select?, fn) })` | same subscription while source (resource) and `select` (identity or source text) are unchanged; `fn` swapped | each event starts a fresh root in the current epoch; `select` is not re-run for a kept subscription | +| `tasks({ name: fn })` | restarted with new code at commit (old one aborted via ctx.signal) | a code holder; calls inside start fresh roots | +| `derive({ name: derivation(sources, fn) })` | kept; fn swapped | synchronous fn | +| `register(registry, { key: value })` | value swapped at commit | eager records only (v1) | +| `expose(resource)` | follows the resource | a local member (usually `opaque`) whose handle value is the resource's value once live | + +State members of a behavior may be cells, derivations, or state members of handles the record holds (they are defined once the record is live). `cell.replace(ctx, value, { diff? })` publishes a structural diff when the new value shares containers with the old one (or `diff: true`), else `r`. Facet `setup` receives `{ serve, name, root, registry }` of its host. + +`migrate` per cell is `(old, fromVersion) => new`. A record may be several versions behind (author bumped more than once between reloads). + +Removed vs guide: `self.own`, `holdWhile`, async factories, inferred dependencies. + +### Synchronous factory, asynchronous activation + +`self.get` and `self.resource` only declare and return placeholders. Activation of a record: + +1. run the factory synchronously (declarations); +2. await declared resources' `open()` (e.g. connect a DB); +3. await declared holds becoming live (dependencies activate first, coalesced); +4. record is live: handles `live`, `ready` resolves. + +Inside the factory, placeholders must not be used (`resource.value` throws `not_ready`; handle calls throw). Methods/listeners/tasks run only on live records, so their dependencies are live. External consumers (UI, roots) holding a `loading` handle: calls wait until live (ctx cancellation, deadline); state members read `undefined` until hydrated. Activation failures propagate to dependents and all waiting gets; the next get retries. + +### Host eager readiness + +- `host.install(facet, { awaitEager? })` installs declarations synchronously and returns a promise for this facet's + eager activations. `host.load(facets, { awaitEager? })` installs the entire batch before awaiting its eager + activations. `awaitEager` defaults to true. Unheld lazy objects are not activated. +- Factory, resource-open, dependency and activation-finalization failures reject the returned promise with the + activation error (ChordError codes preserved; ordinary errors wrapped as `unavailable`). Eager failures also + reach `onError`. Installation is not a rollback transaction; setup/registration errors from `install` throw + synchronously. Abandoned startup should close the host. +- `{ awaitEager: false }` starts the same eager objects but resolves after installation, without waiting for their + activation. `host.ready()` awaits a snapshot of the host's eager holds at call time (including direct eager + `serve` and deferred loads), rejects their activation failures, and does not acquire additional holds. It is not + an ongoing health check and does not await task completion or remote state hydration. +- Local bootstrap readiness and upstream admission are separate. A worker connects its upstream link, loads with + `{ awaitEager: false }`, awaits its local bootstrap handle (e.g. Harness, with no upstream dependency), then + announces control readiness so its parent can accept the link. Only after admission does it await `host.ready()`. + Otherwise an eager reporter depending on server SessionStatus deadlocks with the parent waiting for control + readiness. Deferred loads skip the whole barrier, not selected dependencies; callers explicitly choose local + bootstrap handles, keep bootstrap busy until full readiness, and propagate later startup errors. + +## 4. Principals, variants, handles + +- `createHost({ principalKey })` maps principal to a key that includes connection provenance and grant revision. Records are per principal key. `host` is a built-in principal for in-host calls. +- Remote factories return `{ kind, ...members }`; `kind` must be a declared variant; members must match it. Local factories return any object. +- Handle: `status` (`loading | live | released | gone`), `ready` promise, `kind`, `shape` (`{ kind, members, contract }`), `onStatus(listener)` (every status change, with the error for `gone`), late-bound members, `release()`. A remote handle's state member holds a value only while subscribed (`undefined` after its last subscriber leaves). Shape changes (reload, variant) are pushed; members missing from the current variant reject `not_in_variant`; released handles reject `released`; forced disposal makes handles `gone`. +- Methods are `(args, ctx) => Promise`. Factories still check arguments; variants are not complete authorization. +- Revocation: `host.invalidatePrincipal(match)` force-disposes matching records on this host and propagates over links; links re-stamp or close. + +## 5. Lifetime + +- Hold owners: record (declared `get`), call (`ctx.get`, released at settle), connection (remote holds; relay maps per origin connection), host (eager), UI. +- Activation coalesced per address; if all requesters leave before it completes, the result is disposed. Factory throw: unwind declarations, reject gets, next get retries. +- Disposal after last hold + `linger`: tasks aborted, listeners removed, resources closed (reverse order; all run; errors aggregated), holds released (cascade). +- Bulk forced disposal (`host.close`, `invalidatePrincipal`) goes in waves, holders before what they hold, so a resource's `close` can still use its record's holds; `host.close` closes links after the records. +- Forced disposal: handles `gone` immediately, new calls rejected, in-flight call contexts aborted, then cooperative cleanup. +- Hold graph acyclic in-host (checked at get); across hosts by convention. +- Links have ping/timeout; a dead link releases everything held through it. + +## 6. Roots and epochs + +- `Ctx = { signal, rootId, epoch, get(...) , principal? }`. Explicit; no AsyncLocalStorage dependency (may be used in dev to warn). +- A link creates a root for a request that carries none; nested remote calls carry `{rootId, epoch}`. +- Dispatch: a call with epoch E runs the record's behavior for E. A host retains at most `current` and `previous` epochs (two-version rule). A call for an epoch the host doesn't have (older than previous, or host started later) is rejected `superseded` (retryable by starting a new root). Never substitute another epoch. +- Browsers never carry epochs into the server: browser calls start roots at the server. Browsers learn of changes via shape/reset pushes and a server `Manifest` object; incompatible contract fingerprint -> `upgrade_required`. + +## 7. Reload + +Explicit trigger (`reload_plugins` tool, button, API). Serialized; no nesting. + +1. Build/type-check gate (p2 tooling): dependency-closed set; refuse breaking remote contract changes without a contract version bump. +2. Load: on every affected host, import new facet modules (cache-busted). +3. Prepare: for each live record of each affected type, run the new factory in recording mode: existing cells readable, writes throw; listeners/tasks/registrations recorded, not started; holds recorded; non-exclusive resources whose key is new or `rev`/`deps` changed are opened as candidate-owned (closed on abort). Validate keys, variants, registry collisions. New records activated during prepare use the current epoch (old code). Any failure: abort, discard candidates. Vote. +4. Gate (only records whose state version was bumped or which replace an exclusive resource, plus records depending on those resources): + - Admission cut: from now, an old-epoch root that has not touched the record gets `upgrade_conflict` (retryable), not queued. Roots that already hold an activation lease on it are grandfathered; their re-entry is admitted. + - Wait for grandfathered activations to finish, deadline (default 5 s). Missed deadline: abort the reload, reopen, nothing changed (report blocking roots). + - Exclusive resources: close old, open new; on failure reopen old and abort. + - Run `migrate` into private copies; validate. +5. Commit (server persists the decision, commits itself, then sends commit to children; FIFO links guarantee no host receives a new-epoch root before committing): each host in one synchronous step installs migrated cell values and state versions, installs behaviors as epoch E', swaps registry tables, restarts tasks with new code, rebinds listeners, pushes shape/reset to subscribers, opens gates. E' current, E previous. + The coordinator's result carries `children`: each participating child's prepared report (its vote). +6. Retire: when no roots of `previous` remain (or on the next reload, which first waits for retirement with a deadline), drop its behaviors and release declarations only it declared. + +Compatible changes (no state bump, no exclusive resource change) skip the gate: no waiting. + +State compatibility rule: reload prepare compares each cell's old and new schema. Unchanged: compatible (no gate). Changed with the state version bumped: gate + `migrate`, output validated against the new schema. Changed without a bump: reload refused. A change of meaning with an identical schema needs a bump too (undetectable; author's responsibility). Initial values and migrate outputs are validated; per-write validation in dev mode. + +## 8. Registries + +```ts +export const Routes = defineRegistry("http.routes", { conflict: "error" }); +self.register(Routes, { "GET /download": handler }); // in an eager record; keys are the registry's keys +host.registry(Routes).get("GET /download", ctx); // table of ctx.epoch (or current without ctx) +host.registry(Routes).subscribe((table, epoch) => ...); // bridges for imperative consumers (Durable) +``` + +Per-epoch immutable tables; order = first registration order, kept on replace. Durable bridge: implements Durable's `RegistryReader` from the table (snapshot per epoch) so phases pin a complete snapshot. + +## 9. Hosts and links + +- `createHost({ name, principalKey, upstream? })`; `host.accept(link)` for children; `host.route(prefix, { authorize(principal, hostName), resolve(hostName) => Promise })`. +- Links: `memoryLinkPair()` (tests), Node IPC (child_process), WebSocket (server `ws`, browser `WebSocket`). +- Principal stamping: the accepting side stamps the principal of what connects (from its authenticator); forwarded principals accepted only from the parent link; payload principal fields ignored. +- A child that connects behind its parent's epoch is fenced (nothing routed to it) and reconciled by a reload of its modules to the parent's epoch. Calls its roots of the old epoch make upstream are admitted in the parent's current epoch (the parent never ran that epoch), not rejected `superseded`. +- Relay: server forwards frames for `at` hosts to the resolved child link, mapping holds per origin connection; authorizes before resolving/spawning and re-checks after. +- Wire frames (JSON): `hello`, `get`/`hold`/`release`, `shape`, `call`/`result`/`cancel`, `state` (`snap`/`ops`/`reset` with per-subscription delta encoder), `gone`, `ping`/`pong`, reload control (`prepare`/`vote`/`commit`/`abort`). + +## 10. Host manager (library) + +`createHostManager({ driver, keepAlive })`: fork driver (entry, init, lock, env), serialized start/stop/replace, exit-before-replace, process-tree kill after grace, control channel (ready/fatal/shutdown), liveness callback (booting, activeCalls, viewers with heartbeats, durable tasks, idleFor). + +## 11. Tests required (minimum) + +Lifetime/holds/linger/forced disposal/gone; activation coalescing and abandonment; principals and variants; remote calls over memory links incl. relay and principal forgery attempts; state snap/ops/reset over a link with reconnect; registries per epoch; reload: compatible (no wait, pinned roots), prepare isolation (writes throw, failing factory leaves everything unchanged), state bump with migrate and gate, `upgrade_conflict` for late old roots, deadline abort, exclusive resource replacement, cross-host commit ordering, two-version retirement and `superseded`. diff --git a/packages/chord/docs/tutorial.md b/packages/chord/docs/tutorial.md new file mode 100644 index 000000000..dba37e6e0 --- /dev/null +++ b/packages/chord/docs/tutorial.md @@ -0,0 +1,910 @@ +# Build a shared todo list with Chord + +## 1. What problem does Chord solve? + +A todo list has data, operations on that data, and several people looking at it. +You could hand-write object creation, subscription cleanup, messages between +processes, and code replacement. Chord connects these pieces for you. + +A **host** is the runtime that owns objects. An **object** combines state with +methods at an address, such as “the family todo list.” A **handle** is a caller's +reference to that object, not the object itself. Several handles can reach the +same object. Handles can also reach another host over a connection. + +```text +browser host server host +handle + subscribed rows -- link --> Todos view (owner or guest) + | + handle + v + TodoStore: rows + methods +``` + +We will start inside one host, add observable state, then share it with a +browser. Finally we will add background work, plugins, and hot reload. +**Live state** means subscribers receive new values when data changes, without +asking for it again. + + +```ts +const host = createHost({ name: "todo-server" }); +assert.equal(host.name, "todo-server"); +await host.close(); +``` + +The assertion passes: the host's name is `todo-server`. Creating a host does +not create a todo object, start a server socket, or persist data. `close()` +shuts down its objects and connections. + +> **Tip:** These are executable TypeScript examples, using assertions to show +> results. Shared imports are at the end. Later sections reuse earlier contracts +> and functions, but each section's test starts with fresh hosts. Test-only +> cleanup closes hosts even when an assertion fails; snippets omit that cleanup. + +## 2. Your first object + +Before callers can use an object, they need to agree on its name and members. +That agreement is a **contract**: a runtime value that also gives TypeScript its +types. A **schema** describes a value's shape. We use TypeBox's `Type` to write +schemas; `Type.Null()` means this method takes no useful argument. + +Start with a label for a named list. `local` means only code inside this host +can reach it. The input `{ list: "family" }` identifies which list we want; +it is not an argument to `label`. + + +```ts +const TodoLabel = local("tutorial.todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { label: method(Type.Null(), Type.String()) }, +}); +``` + +A **factory** is a synchronous function that creates the object's behavior: +the methods and state callers will see. `serve` registers a contract and its +factory. Registration alone does not activate this lazy object; the first +`get` does. + +A **root** is one short tree of calls, such as the work for a button click. +`host.root` starts it. Its **context**, called `ctx`, carries cancellation, +caller identity, and the code version for nested calls. Pass it to methods you +call from that root. + + +```ts +const host = createHost(); +host.serve(TodoLabel, ({ list }) => ({ label: () => `Todos for ${list}` })); +const result = await host.root(async (ctx) => { + const todos = ctx.get(TodoLabel, { list: "family" }); + return todos.label(null, ctx); +}); +assert.equal(result, "Todos for family"); +``` + +The result is `"Todos for family"`. `ctx.get` returns a handle immediately; +method calls wait for the object to become live. The handle acquired through +`ctx.get` is released when this root ends. We never receive the factory's +returned object directly. A standalone call without `ctx`, such as +`todos.label(null)`, starts its own root. Inside a root or method, pass the +existing `ctx` so nested calls stay in the same call tree. + +The factory may also receive `principal` (who is asking) and `self` (the +object's declaration API). We did not need them yet. + +> **Why:** The handle separates callers from implementations. Later, replacing +> code will not require replacing every caller's reference. + +## 3. Give the list live state + +A label is not a todo list. We need rows that can change, and a count that always +agrees with them. A **cell** owns an observable JSON value. A **derivation** +computes another observable value from cells or other readable sources. + +Extend the contract with `rows`, `openCount`, and editing methods. State +members describe values, not methods. This adds members at contract version 1; +older callers that only need `label` still have everything they need. + +`self.state` declares named cells. The name `rows` is important: on reload, +Chord matches that name to the existing cell and keeps its value. + + +```ts +const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); +const Rows = Type.Array(Todo); +const TodoStore = local("tutorial.todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { + label: method(Type.Null(), Type.String()), + rows: Rows, + openCount: Type.Number(), + add: method(Type.Object({ text: Type.String() }), Type.Null()), + finish: method(Type.Object({ index: Type.Integer() }), Type.Null()), + reset: method(Rows, Type.Null()), + }, +}); +``` + +The factory now declares a cell and a derivation. Each method receives its +arguments and a context; changing the cell updates its readers and its derived +count together. + + +```ts +const todoFactory: Factory = ({ list }, _principal, self) => { + const state = self.state({ rows: cell(Rows, []) }); + const derived = self.derive({ + openCount: derivation([state.rows], (rows) => rows.filter((todo) => !todo.done).length), + }); + return { + label: () => `Todos for ${list}`, + rows: state.rows, + openCount: derived.openCount, + add: ({ text }, ctx) => { + state.rows.change(ctx, (draft) => { + draft.push({ text, done: false }); + }); + return null; + }, + finish: ({ index }, ctx) => { + state.rows.change(ctx, (draft) => { + const todo = draft[index]; + if (!todo) throw invalid("args", "No such todo"); + todo.done = true; + }); + return null; + }, + reset: (rows, ctx) => { + state.rows.replace(ctx, rows, { diff: true }); + return null; + }, + }; +}; +``` + +`change(ctx, fn)` gives `fn` a temporary mutable **draft**. Here `add` +pushes a row and `finish` changes one field. The draft is discarded if the +callback throws. The callback must be synchronous. + +`replace(ctx, value)` replaces the whole value. We use it for resetting a +list we own completely. `{ diff: true }` asks Chord to publish just the +structural changes. Without that option, replacement diffs automatically when +the new value shares containers at the same keys with the old one; unrelated +values normally publish a whole-value replacement. Either way, subscribers +receive the same resulting value. + + +```ts +const host = createHost({ dev: true }); +host.serve(TodoStore, todoFactory); +const todos = host.get(TodoStore, { list: "family" }); +await todos.ready; +const counts: (number | undefined)[] = []; +const stop = todos.openCount.subscribe((value) => counts.push(value)); +await host.root(async (ctx) => { + await todos.add({ text: "Buy milk" }, ctx); + await todos.finish({ index: 0 }, ctx); + await todos.reset([], ctx); +}); +assert.deepEqual(counts, [0, 1, 0]); +assert.deepEqual(todos.rows.value, []); +stop(); +todos.release(); +``` + +The count subscriber sees `0, 1, 0`: initially empty, one unfinished todo, +then no unfinished todos. Resetting to empty does not add another count +notification because the derived count is still zero. The final rows are `[]`. +`subscribe` calls back immediately with the current value and returns an +unsubscribe function. We await `ready` first so that value is not a loading +placeholder. + +> **Why:** Store reloadable state in cells, not closure variables. Treat +> `cell.value`, `handle.rows.value`, and subscriber values as immutable. +> Only mutate drafts inside `change`; do not keep a draft for later use. +> A derivation runs synchronously, so use it for projections, not database work. + +## 4. Let objects use objects and resources + +Saving a todo may require both the list and a database connection. A **hold** +is a handle's claim that keeps an object alive. `self.get` declares a hold +owned by this object; it lasts longer than a single method call. + +A **resource** is something the runtime opens and later closes, such as a +database connection. Its `open` may be asynchronous even though the factory +must be synchronous. The example database is deliberately fake: it just saves +strings in memory. + +`opaque("name@version")` describes a native, host-only member. It is not +JSON state and cannot be sent over a link. `self.expose` connects that member +to a declared resource, without reading the resource during the factory. + + +```ts +const Database = local("tutorial.database", { + version: 1, + input: Type.Object({}), + members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, +}); +const SavedTodos = local("tutorial.saved-todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { add: method(Type.Object({ text: Type.String() }), Type.Null()) }, +}); +const host = createHost(); +let opens = 0; +const closed = Promise.withResolvers(); +host.serve(Database, (_input, _principal, self) => { + const res = self.resources({ + db: resource({ + rev: 1, + open: async () => { + opens++; + return { saved: [] as string[] }; + }, + close: () => closed.resolve(), + }), + }); + return { db: self.expose(res.db) }; +}); +``` + +The factory returns an exposed marker, not an opened connection. Chord runs +`open` after declarations finish. The promise named `closed` lets the test +observe eventual cleanup. + +Now declare `SavedTodos` as an object that holds both the store and the database. +Its editing method uses these dependencies only after activation. + + +```ts +host.serve(TodoStore, todoFactory); +host.serve( + SavedTodos, + ({ list }, _principal, self) => { + const database = self.get(Database, {}); + const todos = self.get(TodoStore, { list }); + return { + add: async ({ text }, ctx) => { + database.db!.saved.push(text); + await todos.add({ text }, ctx); + return null; + }, + }; + }, + { linger: 0 }, +); +``` + +An outside consumer holds a `SavedTodos` handle. We also hold `Database` +directly so we can inspect what was saved. + + +```ts +const saved = host.get(SavedTodos, { list: "family" }); +const database = host.get(Database, {}); +await saved.ready; +await host.root((ctx) => saved.add({ text: "Buy milk" }, ctx)); +assert.deepEqual(database.db!.saved, ["Buy milk"]); +assert.equal(opens, 1); +const statuses: string[] = []; +saved.onStatus((status) => statuses.push(status)); +database.release(); +saved.release(); +await closed.promise; +assert.equal(saved.status, "released"); +assert.ok(statuses.includes("released")); +``` + +After the method, the fake database contains `["Buy milk"]`. It opened once, +even though two objects held it. `SavedTodos` and `Database` become live +only after their resources and declared dependencies are ready. That is why +`database.db!` is safe inside this live object's method: activation has +already established that dependency. An outside caller must await readiness. + +When both external handles are released, `SavedTodos` is disposed and +releases its own database hold. With no holds left, the database closes; we +wait for `closed.promise` to prove it. The released handle stays +`released`, and `onStatus` reports that transition. `onStatus` watches +changes; unlike state subscription, it does not deliver an initial status. + +`linger` is a delay in milliseconds after the last hold leaves. Zero disposes +without a grace period; a positive value can avoid reopening an expensive +resource when a screen is briefly hidden. Linger is not persistence. + +> **Tip:** A factory only declares dependencies. Never call a held method or +> read `resource.value` there, including during reload. Activation happens +> afterward. Bump a resource's `rev` when its configuration should be reopened. +> Disposal stops tasks and listeners, closes resources, then releases holds. + +## 5. Who is asking? One store, two views + +Owners may add todos; guests may only read them. A **principal** is the +application-defined identity of a caller. A **variant** is a named set of +members a remote object offers to that principal. It lets the UI discover +whether an editing method is available. + +Do not create a separate data cell for each viewer: their lists would diverge. +Instead, each view holds the same host-local `TodoStore` and exposes its rows. +`self.get` defaults to the host principal, so owner and guest views share +that store. Remote view records themselves are separate per principal key. + + +```ts +const Todos = remote("tutorial.todo-view", { + version: 1, + input: Type.Object({ list: Type.String() }), + variants: { + viewer: { state: { rows: Rows } }, + editor: { + state: { rows: Rows }, + methods: { add: method(Type.Object({ text: Type.String() }), Type.Null()) }, + }, + }, +}); + +function serveViews(api: Pick): void { + api.serve(Todos, ({ list }, principal, self) => { + if (list !== "family") throw forbidden("This list is not shared"); + const store = self.get(TodoStore, { list }); + if (principal !== "owner" && principal !== hostPrincipal) { + return { kind: "viewer", rows: store.rows }; + } + return { + kind: "editor", + rows: store.rows, + add: ({ text }, ctx) => { + if (text.trim().length === 0) throw invalid("args", "A todo needs text"); + return store.add({ text }, ctx); + }, + }; + }); +} +``` + +Here principals are the strings `"owner"` and `"guest"`, assigned by +trusted application code. The built-in `hostPrincipal` can also edit. +`principalKey` maps identity to record identity. Real applications should use +stable identity, connection provenance, and grant revision, not display names. + + +```ts +const host = createHost({ principalKey: (principal) => String(principal) }); +host.serve(TodoStore, todoFactory); +serveViews(host); +const owner = host.get(Todos, { list: "family" }, { principal: "owner" }); +const guest = host.get(Todos, { list: "family" }, { principal: "guest" }); +await Promise.all([owner.ready, guest.ready]); +assert.equal(owner.kind, "editor"); +assert.equal(guest.kind, "viewer"); +await host.root((ctx) => owner.add({ text: "Buy milk" }, ctx), { principal: "owner" }); +assert.deepEqual(guest.rows.value, [{ text: "Buy milk", done: false }]); +await assert.rejects(guest.add({ text: "Not allowed" }), (error) => isChordError(error, "not_in_variant")); +await assert.rejects(owner.add({ text: " " }), (error) => isChordError(error, "invalid")); +``` + +The owner gets `editor`; the guest gets `viewer`. Both see the same milk +todo. Calling the guest's `add` rejects with `not_in_variant`, even though +TypeScript exposes methods from all variants on a handle. Whitespace-only text +rejects with `invalid`: it is a string, as the schema demands, but it breaks a +rule only the method knows. Use `invalid` for bad input and `forbidden` only +for callers who may not do something. + +> **Why:** A variant is not all of authorization. The factory checks which +> list may be opened; the method checks the operation's arguments. Production +> code must also enforce grants that can change. Use +> `host.invalidatePrincipal(match)` to revoke matching records; held handles +> become `gone`. A UI watches `onStatus` and gets a new handle when appropriate. +> Guest-provided names or message text must never establish identity. + +## 6. Reach the list from another process + +A browser cannot call a host-only contract. `remote` contracts describe only +schema-checked state and methods that can travel as JSON. Our `Todos` view is +remote; the underlying `TodoStore` and database stay local. + +A **link** carries ordered messages between hosts. A memory link runs both ends +in one test process, but uses the remote protocol. The accepting host stamps +the caller's principal from its authenticator, not from method arguments or a +client's claimed identity. + + +```ts +const server = createHost({ name: "server" }); +server.serve(TodoStore, todoFactory); +serveViews(server); +const [down, up] = memoryLinkPair(); +server.accept(down, { principal: "owner", kind: "client" }); +const browser = createHost({ name: "browser", kind: "client", upstream: up }); +const todos = browser.get(Todos, { list: "family" }, { at: "server" }); +const hydrated = Promise.withResolvers(); +const updated = Promise.withResolvers(); +const stop = todos.rows.subscribe((rows) => { + if (rows !== undefined) hydrated.resolve(); + if (rows?.length === 1) updated.resolve(); +}); +await todos.ready; +await hydrated.promise; +assert.deepEqual(todos.rows.value, []); +await browser.root((ctx) => todos.add({ text: "Buy milk" }, ctx)); +await updated.promise; +assert.deepEqual(todos.rows.value, [{ text: "Buy milk", done: false }]); +// Deliberately bypass TypeScript to simulate a malformed caller. +await assert.rejects(todos.add({ text: 123 } as never), (error) => isChordError(error, "invalid")); +stop(); +assert.equal(todos.rows.value, undefined); +todos.release(); +``` + +The browser sees an empty snapshot, then the new milk todo. `ready` means +the remote handle has bound to a live object; it does **not** guarantee that +subscribed state has arrived. That is why we separately await `hydrated` and +`updated`. A remote readable starts at `undefined` and returns to +`undefined` when its last subscriber leaves. + +What crosses the wire: the input and consumer's contract description, the +provider's shape (variant and members), method arguments and results, and +subscribed state. State starts with a **snapshot**, the whole current value; +subsequent **deltas** describe changes. Rebinds and reloads can send a +`reset`, a fresh whole value. Functions, resource connections, and cell +instances do not cross the wire. + +The deliberate bad argument rejects with `invalid`. Remote arguments, +results, and outgoing state are validated at the provider boundary. Local +methods do not get per-call schema validation; they still need app-level checks. +`dev: true` additionally checks cell writes while developing. + +> **Tip:** Replace `memoryLinkPair` with `webSocketLink` for a WebSocket or +> `ipcLink` for a child-process IPC channel; object contracts stay the same. +> Those adapters wrap an existing connection, not an authenticator. Configure +> `accept` with an authenticated principal. A browser is `kind: "client"`; +> a trusted child host is accepted with `kind: "host"` and its configured name. + +## 7. React to changes and run background work + +Not all work comes from method calls. The app may update an activity indicator +when rows change, or run a background reminder. A **listener** reacts to an +observable source. A **task** is background execution owned by an object, +which is aborted when the object is disposed or its code is reloaded. + +Our activity object mirrors the row count. Its task inserts a sample todo, +then waits for shutdown. The promises make those events deterministic in a +test; they are not stored application state. + + +```ts +const Activity = local("tutorial.activity", { + version: 1, + input: Type.Object({}), + members: { seen: Type.Number() }, +}); +const host = createHost(); +host.serve(TodoStore, todoFactory); +const observed = Promise.withResolvers(); +const seeded = Promise.withResolvers(); +const stopped = Promise.withResolvers(); +host.serve(Activity, (_input, _principal, self) => { + const todos = self.get(TodoStore, { list: "family" }); + const state = self.state({ seen: cell(Type.Number(), 0) }); + self.listen({ + rows: listener(todos.rows, (rows, ctx) => { + state.seen.replace(ctx, rows?.length ?? 0); + if (rows?.length === 1) observed.resolve(); + }), + }); + self.tasks({ + seed: async (ctx) => { + await todos.add({ text: "Review the list" }, ctx); + seeded.resolve(); + await new Promise((resolve) => { + if (ctx.signal.aborted) resolve(); + else ctx.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + stopped.resolve(); + }, + }); + return { seen: state.seen }; +}); +const activity = host.get(Activity, {}); +await activity.ready; +await Promise.all([seeded.promise, observed.promise]); +assert.equal(activity.seen.value, 1); +activity.release(); +await stopped.promise; +``` + +The final `seen` is `1`. Each listener event receives a fresh root's +`ctx`; its write belongs to that short call tree. A task is not one enormous +root: Chord method calls made with its task context start fresh short roots. +We pass that context into `todos.add` so cancellation follows the task. +Releasing the activity handle aborts its task and resolves `stopped`. + +Listeners can also subscribe to a resource's event stream with +`listener(resource, select, fn)`: `select` gets the opened resource and +returns something with `subscribe`. Kept subscriptions survive reload while +the resource and selector remain unchanged; the callback switches to new code. +A selector that captures changing configuration should get a new listener name. + +> **Why:** Do not wait for a person, sleep for minutes, or run an endless loop +> inside a method. That pins a root and can block state migration. Store a +> pending request in a cell and answer in another call, or use a task that obeys +> `ctx.signal`. Never retain a completed method's `ctx` for background work. + +## 8. Package features as plugins + +Now a plugin can contribute a menu action without editing a central switch. +A **registry** is a named table of contributions. A **facet** declares what a +plugin serves on one kind of host. A plugin can have server, worker, and browser +facets, each installing that part of the feature. + +Registrations belong to eager objects: **eager** means the host creates and +holds the object at startup, even if no screen asks for it. That keeps our menu +contribution present. Lazy objects still activate only when held. + + +```ts +const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); +const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); +let click: (() => Promise) | undefined; +const storeFacet: Facet = { + name: "todo-store", + ...defineFacet((api) => api.serve(TodoStore, todoFactory)), +}; +const menuFacet: Facet = { + name: "todo-menu", + ...defineFacet( + (api) => { + api.serve( + Menu, + (_input, _principal, self) => { + const todos = self.get(TodoStore, { list: "family" }); + self.register(Actions, { + addSample: async (ctx) => { + await todos.add({ text: "Try plugins" }, ctx); + return todos.openCount.value!; + }, + }); + return {}; + }, + { eager: true }, + ); + click = () => api.root((ctx) => api.registry(Actions).get("addSample", ctx)!(ctx)); + }, + { uses: [TodoStore] }, + ), +}; +const host = createHost(); +await host.load([menuFacet, storeFacet], { awaitEager: false }); +await host.ready(); +assert.equal(await click!(), 1); +assert.equal(host.registry(Actions).table().size, 1); +``` + +Clicking returns `1`, and the registry has one entry. The menu's dependency +can come from a later facet in the load batch: `load` installs the whole +batch before waiting for eager activation. The facet declares `uses` so +reload can also check the contract it consumes. + +The facet API supplies `serve`, `name`, `root`, and `registry`. +Our ordinary UI callback uses its saved `api.root`, not a process-global +host. Looking up a registry entry with `ctx` uses the table belonging to that +root's code version. Without `ctx`, lookup uses the current table. Duplicate +keys fail by default; `subscribe` can bridge tables to an imperative consumer. + +Normally use `await host.load(facets)` (or `await host.install(facet)`), +which waits for those eager objects. This example explicitly defers that barrier +with `awaitEager: false`, then awaits `host.ready()`. That waits for a +snapshot of the host's eager holds, not for task completion or later loads. +Failures reject the readiness promise; abandoning startup requires closing the +host, because installation is not a rollback transaction. + +> **Tip:** Use deferred readiness only when needed. A worker may have to await +> an independent local bootstrap handle, announce control readiness so its +> parent admits its link, and only then await `host.ready()`. Waiting for +> upstream-dependent eager objects before admission would deadlock. + +For real plugin files, default-export `defineFacet(...)`. Node hosts configure +`createHost({ loader: nodeLoader })`, with `nodeLoader` imported from +`@earendil-works/chord/node`, then pass module URLs to `load` and `reload`. +The in-memory facet objects here avoid filesystem setup. Built-in facets are +trusted in-process code, not a sandbox; the current Node loader keeps old module +copies in memory, so repeated loading has that limitation. + +## 9. Change running code without losing the list + +A handle is useful precisely because callers keep it while implementations +change. An **epoch** identifies one committed set of code and registry tables. +A root keeps the epoch it started with. New roots use the latest one. + +### Compatible code change: no waiting + +Give the same named facet a new factory. It changes the label but keeps the +cell declarations. This test deliberately pauses one caller to show what an +old root sees; application methods should not pause like this. + + +```ts +const host = createHost(); +const facet = (label: string): Facet => ({ + name: "todos", + setup: (api) => + api.serve(TodoStore, (input, principal, self) => ({ + ...todoFactory(input, principal, self), + label: () => label, + })), +}); +await host.load([facet("Original")]); +const todos = host.get(TodoStore, { list: "family" }); +await todos.add({ text: "Buy milk" }); +const entered = Promise.withResolvers(); +const resume = Promise.withResolvers(); +const oldRoot = host.root(async (ctx) => { + const first = await todos.label(null, ctx); + entered.resolve(); + await resume.promise; // Test-only pause: pin an old caller during reload. + return [first, await todos.label(null, ctx)]; +}); +await entered.promise; +const compatible = await host.reload([facet("Updated")]); +assert.deepEqual(compatible.gated, []); +assert.equal(await todos.label(null), "Updated"); +resume.resolve(); +assert.deepEqual(await oldRoot, ["Original", "Original"]); +assert.equal(todos.rows.value?.[0]?.text, "Buy milk"); +``` + +Reload commits while the old root is paused: `gated` is empty. New calls see +`"Updated"`; the resumed old root sees `"Original"` twice. The milk row +survives. Compatible code changes do not wait for that old root to finish. + +### State change: bump its version and migrate + +Suppose a new policy converts existing todo text to uppercase. The schema is +unchanged, but the data's meaning changes. We still bump the **state version**. +This version belongs to stored cells, independently of the contract version. +A **migration** converts an old cell value to the new representation. + + +```ts +const events: string[] = []; +const stopEvents = todos.rows.events((event) => events.push(event.type)); +const migrated = await host.reload([ + { + name: "todos", + setup: (api) => + api.serve(TodoStore, todoFactory, { + state: { + version: 2, + migrate: { + rows: (old, from) => { + assert.equal(from, 1); + return (old as Static).map((todo) => ({ ...todo, text: todo.text.toUpperCase() })); + }, + }, + }, + }), + }, +]); +assert.equal(migrated.gated.length, 1); +assert.deepEqual(todos.rows.value, [{ text: "BUY MILK", done: false }]); +assert.deepEqual(events, ["snap", "reset"]); +stopEvents(); +``` + +The migration receives the old rows and their state version, `1`. It returns +new rows; the handle now reads `[{ text: "BUY MILK", done: false }]`. +Subscribers get a `reset` event containing the migrated value. The event +assertion sees an initial `snap`, then that `reset`. +One record is gated. A **gate** waits for calls already inside that record to +finish before migrating a private copy, validating it, and committing. Old +roots that had not entered it cannot later use old code over migrated data: +they get `upgrade_conflict` and should retry in a new root. + +Changing a cell's schema also requires a state-version bump and a migration +whose result fits the new schema. A change of meaning with the same schema, +like this example, cannot be detected automatically. Real migrations must +handle each stored version they support, not assume every installation is +exactly one version behind. + +### Contract change: keep an adapter for old callers + +Rename `label` to `heading`. That is a breaking contract change, so use +contract version 2. The provider's `accepts[1]` adapter builds the behavior +that version-1 consumers still expect. + + +```ts +// Remove label from the new provider's contract: it is now called heading. +const { label: _label, ...members } = TodoStore.def.members; +const RenamedStore = local("tutorial.todos", { + version: 2, + input: TodoStore.def.input, + members: { ...members, heading: method(Type.Null(), Type.String()) }, +}); +await host.reload([ + { + name: "todos", + setup: (api) => + api.serve( + RenamedStore, + (input, principal, self) => { + const { label, ...behavior } = todoFactory(input, principal, self); + return { ...behavior, heading: label }; + }, + { + state: { version: 2 }, + accepts: { + 1: (current) => ({ ...current, label: current.heading }), + }, + }, + ), + }, +]); +assert.equal(await todos.label(null), "Todos for family"); +const newer = host.get(RenamedStore, { list: "family" }); +assert.equal(await newer.heading(null), "Todos for family"); +assert.equal(todos.rows.value?.[0]?.text, "BUY MILK"); +``` + +The old handle's `label` and the new handle's `heading` both return +`"Todos for family"`. Both still reach the migrated rows. The adapter maps +behaviors, not handles; forwarding the existing method preserves its context. +We keep state version 2 here so this code-only change does not repeat migration. + +> **Why:** Contract versions protect callers; state versions protect stored +> data; resource revisions control reopening. Do not use one number for all +> three. Additive compatible contracts can keep their version; breaking shape +> or meaning changes need a contract bump and adapters for old consumers. +> Reload prepares candidates first; preparation failure leaves the old code +> running. A migration gate that misses its deadline aborts rather than +> switching data underneath active old code. + +## 10. Test the objects, not a whole deployment + +Most behavior needs only a bare host. Use real factories for state tests, and a +fake facet when you want to test delegation without a real database or service. +Here the fake records what the editing view sends to the store. + + +```ts +const host = createHost({ dev: true }); +const received: string[] = []; +const fake: Facet = { + name: "fake-store", + ...defineFacet((api: FacetApi) => { + api.serve(TodoStore, (input, principal, self) => ({ + ...todoFactory(input, principal, self), + add: ({ text }) => { + received.push(text); + return null; + }, + })); + }), +}; +await host.load([fake]); +serveViews(host); +const owner = host.get(Todos, { list: "family" }, { principal: "owner" }); +await owner.add({ text: "Test delegation" }); +assert.deepEqual(received, ["Test delegation"]); +await assert.rejects(owner.add({ text: "" }), (error) => isChordError(error, "invalid")); +owner.release(); +await assert.rejects(owner.add({ text: "Too late" }), (error) => isChordError(error, "released")); +``` + +The fake received `["Test delegation"]`. Empty text failed before reaching +the fake. The released handle rejects further calls with `released`. +`isChordError(error, code)` checks a stable error code; avoid testing only a +message's wording. + +Our other tests exercise the real cells (section 3), resource cleanup (4), +principal variants (5), a client over a memory link (6), and callers held through +reload (9). For remote state tests, await the value you need, not just +`handle.ready`. Always unsubscribe and close hosts; leaked holds can hide +lifetime bugs or leave background tasks running. + +> **Tip:** A fake must honor the contract it serves. This fake delegates all +> ordinary members to `todoFactory` and replaces only `add`; it deliberately +> records calls rather than updating rows. Do not confuse that with persistence. + +## 11. Keep these habits + +A temporary call handle should not accidentally become a permanent hold. A +screen should not lose its data merely because a button click finishes. Use +`ctx.get` for the first lifetime and `host.get` for the second. + + +```ts +const host = createHost(); +host.serve(TodoStore, todoFactory); +const screen = host.get(TodoStore, { list: "family" }); +await screen.ready; +await host.root(async (ctx) => { + const temporary = ctx.get(TodoStore, { list: "family" }); + await temporary.add({ text: "Keep roots short" }, ctx); +}); // The temporary handle is released here; screen still holds the store. +assert.equal(screen.openCount.value, 1); +screen.release(); // No hold remains; without persistence, disposal loses these rows. +``` + +The screen still sees count `1` after the root finishes, because it owns a +separate hold. When the screen releases, no hold remains. This tutorial's cells +are in memory: reload preserves a live record, but disposal or process restart +loses its rows. Real persistence belongs in a database resource, with an +explicit load/save policy. + +### Checklist and common mistakes + +- Keep reloadable state in cells, not closure variables: factories rerun. +- Change drafts, not published values: mutation outside cells bypasses updates. +- Use `change` for field edits and `replace` for owned snapshots: replacing + can erase fields your code does not know about. +- Declare dependencies in a synchronous factory, not during methods: activation + and reload need a complete declaration list. +- Use `ctx.get` for temporary method dependencies: their holds end with the call. +- Pass `ctx` into nested calls: cancellation and code versions must stay together. +- Keep roots short and tasks cancellable: migration and shutdown need work to settle. +- Check grants and arguments, not just variants: available methods are not a full policy. +- Stamp principals at trusted ingress, not from client payloads: names are not identities. +- Subscribe before expecting remote state: readiness is not snapshot hydration. +- Release handles and unsubscribe: otherwise unused objects remain held or observed. +- Bump state versions, contract versions, and resource revisions for their own changes: + each protects a different boundary. +- Await eager readiness before reporting startup success: declared is not yet live. +- Keep hold dependencies acyclic: an object cannot wait for itself through another object. + +### Imports and running the examples + +These are the shared imports from `test/tutorial.test.ts`. The relative +path is correct from that test directory; applications import the same names +from `@earendil-works/chord`. `assert` expresses expected values; `it` and +`onTestFinished` come from the test runner; `readFileSync` is used by the +tutorial's source-matching check, not by the application. + + +```ts +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { type Static, Type } from "typebox"; +import { it, onTestFinished } from "vitest"; +import { + type Ctx, + cell, + createHost, + defineFacet, + defineRegistry, + derivation, + type Facet, + type FacetApi, + type Factory, + forbidden, + type Host, + hostPrincipal, + invalid, + isChordError, + listener, + local, + memoryLinkPair, + method, + opaque, + remote, + resource, +} from "../src/index.ts"; +``` + +From `packages/chord`, run: + +```sh +node ../../node_modules/vitest/dist/cli.js --run test/tutorial.test.ts +``` + +There is one test per numbered section. Every TypeScript block is matched +against its runnable source, so changing either copy without the other fails a +test. The test harness adds teardown and drift checks; it never uses a real +database, network service, or provider API. + +Next read [the usage guide](guide.md) for the API reference and +[the specification](spec.md) for compatibility, lifetime, and reload +rules. Implementation details and current limitations are in +[notes.md](notes.md). If a reference disagrees with +`src`, the current implementation wins. diff --git a/packages/chord/package.json b/packages/chord/package.json index 36080e587..29afec869 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -1,7 +1,7 @@ { "name": "@earendil-works/chord", "version": "1.1.0", - "description": "Application composition runtime for services, replicated state, RPC, and plugins", + "description": "Typed host runtime for plugins, contracts, replicated state, and RPC over JSON links", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -21,11 +21,6 @@ "types": "./dist/delta/index.d.ts", "import": "./dist/delta/index.js" }, - "./bundler": { - "source": "./src/bundler.ts", - "types": "./dist/bundler.d.ts", - "import": "./dist/bundler.js" - }, "./node": { "source": "./src/node.ts", "types": "./dist/node.d.ts", @@ -36,6 +31,7 @@ "sideEffects": false, "files": [ "dist", + "docs", "README.md", "src/delta/README.md" ], @@ -49,7 +45,7 @@ "application", "plugins", "rpc", - "services", + "contracts", "state" ], "author": "Earendil Works", @@ -63,7 +59,7 @@ "node": ">=22.19.0" }, "dependencies": { - "esbuild": "0.28.2" + "typebox": "1.3.27" }, "devDependencies": { "shx": "0.3.4", diff --git a/packages/chord/src/api.ts b/packages/chord/src/api.ts deleted file mode 100644 index b566965c1..000000000 --- a/packages/chord/src/api.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { FacetKernel } from "./facets/host.ts"; -import { disposeLoadedFacets } from "./facets/loader.ts"; -import { RemoteServiceBindingImpl } from "./services/consumer.ts"; -import { attachReplicatedStateSource, MutableReplicatedStateImpl } from "./services/state.ts"; -import type { - AttachedReplicatedState, - Facet, - FacetHost, - FacetLoader, - FacetOptions, - LoadedFacets, - MutableReplicatedState, - RemoteServiceBinding, - RemoteServiceBindingOptions, - RemoteServiceContract, - ReplicatedStateSource, - ReplicatedStateSourceOptions, - Service, -} from "./types.ts"; - -/** Create an active host for one complete set of facets. */ -export async function createFacetHost(options: FacetOptions): Promise { - const kernel = new FacetKernel(options); - await kernel.activate(); - return Object.freeze({ - services: kernel.provider, - reload: (facets: readonly Facet[]) => kernel.reload(facets), - dispose: () => kernel.dispose(), - }); -} - -export function createStaticFacetLoader(facets: readonly Facet[]): FacetLoader { - const loadedFacets = Object.freeze([...facets]); - return { - async load() { - return { facets: loadedFacets, async dispose() {} }; - }, - }; -} - -export function combineFacetLoaders(loaders: readonly FacetLoader[]): FacetLoader { - return { - async load() { - const loaded: LoadedFacets[] = []; - try { - for (const loader of loaders) loaded.push(await loader.load()); - } catch (error) { - const cleanupErrors = await disposeLoadedFacets(loaded.reverse()); - if (cleanupErrors.length > 0) { - throw new AggregateError([error, ...cleanupErrors], "Facet loading and cleanup failed"); - } - throw error; - } - let disposed = false; - return { - facets: Object.freeze(loaded.flatMap(({ facets }) => facets)), - async dispose() { - if (disposed) return; - disposed = true; - const errors = await disposeLoadedFacets([...loaded].reverse()); - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, "Failed to dispose loaded facets"); - }, - }; - }, - }; -} - -export function defineFacet(facet: Facet): Facet { - return facet; -} - -export function defineService(id: string, options: { readonly local: true }): Service; -export function defineService( - id: string, - ...options: [RemoteServiceContract] extends [never] - ? readonly [options: never] - : readonly [options?: { readonly local?: false }] -): Service; -export function defineService(id: string, options?: { readonly local?: boolean }): Service { - if (id.length === 0) throw new TypeError("Service ID must not be empty"); - // TODO: check if the reserved namespace should be part of Chord. - if (id.startsWith("$chord.")) throw new TypeError("Service IDs beginning with $chord. are reserved"); - return Object.freeze({ id, local: options?.local ?? false }); -} - -export function createRemoteServiceBinding(options: RemoteServiceBindingOptions): RemoteServiceBinding { - return new RemoteServiceBindingImpl(options); -} - -export function replicatedState( - source: ReplicatedStateSource, - options?: ReplicatedStateSourceOptions, -): AttachedReplicatedState; -/** - * Create authoritative state by taking immutable ownership of an alias-free strict-JSON root. - * The caller must not mutate `initial` after this call. - */ -export function replicatedState(initial: T): MutableReplicatedState; -export function replicatedState( - initialOrSource: object | ReplicatedStateSource, - options?: ReplicatedStateSourceOptions, -): MutableReplicatedState | AttachedReplicatedState { - if (isReplicatedStateSource(initialOrSource)) return attachReplicatedStateSource(initialOrSource, options); - return new MutableReplicatedStateImpl(initialOrSource); -} - -function isReplicatedStateSource(value: object): value is ReplicatedStateSource { - return typeof (value as { readonly attach?: unknown }).attach === "function"; -} diff --git a/packages/chord/src/bundler.ts b/packages/chord/src/bundler.ts deleted file mode 100644 index ef60ebc2a..000000000 --- a/packages/chord/src/bundler.ts +++ /dev/null @@ -1,9 +0,0 @@ -export type { - BundleFacetsOptions, - BundleFacetsResult, - FacetBundlePlatform, -} from "./node/bundle.ts"; -export { bundleFacets } from "./node/bundle.ts"; -export type { FacetBundleEntry, FacetBundleManifest } from "./node/manifest.ts"; -export type { BundleFacetPackageOptions, BundleFacetPackageResult } from "./node/package.ts"; -export { bundleFacetPackage } from "./node/package.ts"; diff --git a/packages/chord/src/cell.ts b/packages/chord/src/cell.ts new file mode 100644 index 000000000..cbfac983b --- /dev/null +++ b/packages/chord/src/cell.ts @@ -0,0 +1,352 @@ +import type { TSchema } from "typebox"; +import { type Draft, diffRevisions, type JsonValue, type Op, type Tracker, track } from "./delta/index.ts"; +import { ChordError } from "./errors.ts"; +import { enqueue, holdPublications } from "./publish.ts"; +import { assertValid } from "./schema.ts"; +import type { Ctx, Readable, StateEvent } from "./types.ts"; + +export interface SourceOptions { + readonly dev: boolean; + readonly report: (error: unknown) => void; +} + +/** Checks that `ctx` may write a record's cells now; throws otherwise. */ +export type WriteGuard = (ctx: Ctx) => void; + +let sequence = 0; + +/** Fans out values and delta events; listener errors are reported, not thrown to the writer. */ +class Fanout { + readonly #values = new Set<(value: T) => void>(); + readonly #events = new Set<(event: StateEvent) => void>(); + readonly #report: (error: unknown) => void; + + constructor(report: (error: unknown) => void) { + this.#report = report; + } + + subscribe(listener: (value: T) => void, current: T): () => void { + this.#values.add(listener); + this.#call(listener, current); + return () => this.#values.delete(listener); + } + + events(listener: (event: StateEvent) => void, current: T): () => void { + this.#events.add(listener); + this.#call(listener, { type: "snap", value: current }); + return () => this.#events.delete(listener); + } + + /** + * Queues delivery to the listeners subscribed now. A listener that subscribes later already got this revision in + * its snapshot; one that unsubscribes before delivery is skipped. + */ + emit(event: StateEvent): void { + const events = [...this.#events]; + const values = [...this.#values]; + enqueue(() => { + for (const listener of events) if (this.#events.has(listener)) this.#call(listener, event); + for (const listener of values) if (this.#values.has(listener)) this.#call(listener, event.value); + }); + } + + #call(listener: (value: V) => void, value: V): void { + try { + listener(value); + } catch (error) { + this.#report(error); + } + } +} + +/** A record-owned state source (cell or derivation). Writes recompute dependents atomically. */ +export abstract class Source implements Readable { + readonly key: string; + readonly seq = ++sequence; + readonly dependents = new Set>(); + /** Schemas of remote state members currently exposing this source (checked per write in dev mode). */ + schemas: readonly TSchema[] = []; + protected readonly options: SourceOptions; + protected readonly fanout: Fanout; + + constructor(key: string, options: SourceOptions) { + this.key = key; + this.options = options; + this.fanout = new Fanout(options.report); + } + + abstract get value(): T; + + subscribe(listener: (value: T) => void): () => void { + return this.fanout.subscribe(listener, this.value); + } + + events(listener: (event: StateEvent) => void): () => void { + return this.fanout.events(listener, this.value); + } + + /** Per-write validation, dev mode only. */ + validate(value: unknown): void { + if (!this.options.dev) return; + for (const schema of this.ownSchemas()) assertValid("state", `state ${this.key}`, schema, value); + for (const schema of this.schemas) assertValid("state", `state ${this.key}`, schema, value); + } + + protected ownSchemas(): readonly TSchema[] { + return []; + } +} + +/** The tracker tracks containers, so a cell tracks `{ v: value }` and strips the box from published ops. */ +interface Box { + v: T; +} + +function unbox(ops: readonly Op[], value: unknown): Op[] { + const result: Op[] = []; + for (const op of ops) { + if (op[0] === "r" || op[1].length === 0) return [["r", value as JsonValue]]; + const path = op[1].slice(1); + switch (op[0]) { + case "p": + result.push(["p", path, op[2], op[3], op[4]]); + break; + case "m": + result.push(["m", path, op[2]]); + break; + default: + // s/d/a/t on the value itself replace it. + if (path.length === 0) return [["r", value as JsonValue]]; + if (op[0] === "d") result.push(["d", path as [string | number, ...(string | number)[]]]); + else result.push([op[0], path as [string | number, ...(string | number)[]], op[2]] as Op); + } + } + return result; +} + +/** A state cell holding any JSON value, validated against its schema at creation and (dev mode) on writes. */ +export class Cell extends Source { + /** Replaced at a reload commit that changes the schema (state-version bump). */ + schema: TSchema; + readonly #tracker: Tracker>; + readonly #guard: WriteGuard; + + /** `initial` is always validated against `schema`; ownership moves to the cell. */ + constructor(key: string, schema: TSchema, initial: T, guard: WriteGuard, options: SourceOptions) { + super(key, options); + assertValid("state", `initial value of state ${key}`, schema, initial); + this.schema = schema; + this.#tracker = track>({ v: initial }); + this.#guard = guard; + } + + protected override ownSchemas(): readonly TSchema[] { + return [this.schema]; + } + + get value(): T { + return this.#tracker.value.v; + } + + /** + * Mutates a draft (only mutated paths are published), or returns a new value (`n => n + 1`). A returned value is + * always validated against the schema. The draft is unusable after `fn` returns. + */ + change(ctx: Ctx, fn: (draft: Draft) => T): void; + change(ctx: Ctx, fn: (draft: Draft) => void): void; + change(ctx: Ctx, fn: (draft: Draft) => unknown): void { + this.#guard(ctx); + const change = this.#tracker.beginChange(); + let prepared: ReturnType; + try { + const draft = change.state as Draft>; + const result: unknown = fn(draft.v as Draft); + if (result instanceof Promise) throw new TypeError("cell.change callbacks must be synchronous"); + if (result !== undefined) { + if (result !== draft.v) assertValid("state", `state ${this.key}`, this.schema, result); + draft.v = result as Draft; + } + prepared = change.prepare(); + } catch (error) { + change.abort(); + throw error; + } + this.#publish(prepared, "ops"); + } + + /** + * Replaces the whole value. Ownership of `value` moves to the cell: never mutate it afterwards. + * + * Subscribers get a structural diff (`diffRevisions`) when `diff` is true, or by default when the new value shares + * containers with the old one (a new immutable revision of the same state, e.g. mirrored from another store); + * otherwise a single `r` op. The resulting value is the same either way; only the ops differ. + */ + replace(ctx: Ctx, value: T, options: { readonly diff?: boolean } = {}): void { + this.#guard(ctx); + const before = this.value; + const diff = options.diff ?? sharesStructure(before, value); + const ops = diff ? diffRevisions(before as JsonValue, value as JsonValue) : undefined; + this.#publish(this.#tracker.prepareReplace({ v: value }), "ops", true, ops); + } + + /** + * Installs a migrated value and its schema (reload commit). The caller validated the value and the derivations + * over it against every schema that will apply, so nothing is checked here. Subscribers receive a reset. + */ + reset(value: T, schema: TSchema): void { + this.schema = schema; + this.#publish(this.#tracker.prepareReplace({ v: value }), "reset", false); + } + + /** Adopts this revision and every derived one, then queues all notifications (cell first, then derivations). */ + #publish( + prepared: ReturnType>["prepareReplace"]>, + type: "ops" | "reset", + validate = true, + ops?: readonly Op[], + ): void { + if (prepared.ops.length === 0 && type === "ops") { + this.#tracker.adopt(prepared); + return; + } + const value = prepared.value.v; + let candidates: Map, unknown>; + try { + if (validate) this.validate(value); + candidates = computeDependents(this as Source, value, validate); + } catch (error) { + prepared.abort(); + throw error; + } + holdPublications(() => { + this.#tracker.adopt(prepared); + const published = ops !== undefined && ops.length > 0 ? ops : unbox(prepared.ops, value); + this.fanout.emit(type === "reset" ? { type, value } : { type, ops: published, value }); + for (const [derived, next] of candidates) derived.install(next); + }); + } +} + +/** True if `b` reuses at least one container of `a` at the same key: a new revision of the same state. */ +function sharesStructure(a: unknown, b: unknown): boolean { + if (a === null || b === null || typeof a !== "object" || typeof b !== "object" || a === b) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + for (const [key, child] of Object.entries(b)) { + if (child !== null && typeof child === "object" && (a as Record)[key] === child) return true; + } + return false; +} + +/** Recomputes every transitive dependent of `changed` against candidate values and validates them. */ +function computeDependents(changed: Source, value: unknown, validate = true): Map, unknown> { + const pending = new Set>(); + const visit = (source: Source): void => { + for (const derived of source.dependents) { + if (pending.has(derived)) continue; + pending.add(derived); + visit(derived); + } + }; + visit(changed); + const values = new Map, unknown>([[changed, value]]); + const result = new Map, unknown>(); + for (const derived of [...pending].sort((a, b) => a.seq - b.seq)) { + const next = derived.compute((source) => (values.has(source) ? values.get(source) : source.value)); + if (validate) derived.validate(next); + values.set(derived, next); + result.set(derived, next); + } + return result; +} + +export type Sources = readonly Readable[]; +export type SourceValues = { [K in keyof S]: S[K] extends Readable ? T : never }; + +const isReadable = (value: unknown): value is Readable => + value !== null && + typeof value === "object" && + typeof (value as Readable).subscribe === "function" && + "value" in value; + +/** + * A synchronous derivation. Sources are record cells/derivations (recomputed atomically with the write) or other + * readables such as handle state members (recomputed when they publish). Subscribers receive diffs. + */ +export class Derived extends Source { + readonly sources: readonly Readable[]; + #fn: (...values: never[]) => unknown; + #value: T; + #external: (() => void)[] = []; + + constructor(key: string, sources: Sources, fn: (...values: never[]) => unknown, options: SourceOptions) { + super(key, options); + this.sources = sources.map((source) => { + if (!isReadable(source)) throw new TypeError(`derive ${key}: sources must be readables`); + return source; + }); + this.#fn = fn; + this.#value = undefined as T; + } + + get value(): T { + return this.#value; + } + + /** + * Computes the first value (or takes it from `precomputed`, a reload's validated dry run) and starts following + * the sources (at install, never inside a factory). + */ + attach(precomputed?: ReadonlyMap): void { + this.#value = precomputed?.has(this) ? (precomputed.get(this) as T) : this.compute((source) => source.value); + for (const source of this.sources) { + if (source instanceof Source) source.dependents.add(this as Derived); + else this.#external.push(source.subscribe(() => this.refresh())); + } + } + + detach(): void { + for (const source of this.sources) + if (source instanceof Source) source.dependents.delete(this as Derived); + for (const unsubscribe of this.#external.splice(0)) unsubscribe(); + } + + compute(read: (source: Readable) => unknown, fn: (...values: never[]) => unknown = this.#fn): T { + const result: unknown = fn(...(this.sources.map(read) as never[])); + if (result instanceof Promise) throw new TypeError(`derive ${this.key}: fn must be synchronous`); + return result as T; + } + + /** Replaces the function without recomputing (reload commit, before migrated cells are installed). */ + setFn(fn: (...values: never[]) => unknown): void { + this.#fn = fn; + } + + /** Recomputes and republishes, including dependents. Errors are reported (the source write already happened). */ + refresh(): void { + try { + const next = this.compute((source) => source.value); + const dependents = computeDependents(this as Derived, next); + holdPublications(() => { + this.install(next); + for (const [derived, value] of dependents) derived.install(value); + }); + } catch (error) { + this.options.report(error); + } + } + + install(next: T): void { + const before = this.#value; + this.#value = next; + const ops: Op[] = + before === undefined || next === undefined + ? [["r", (next ?? null) as JsonValue]] + : diffRevisions(before as JsonValue, next as JsonValue); + if (ops.length > 0) this.fanout.emit({ type: "ops", ops, value: next }); + } +} + +export const isSource = (value: unknown): value is Cell | Derived => value instanceof Source; + +/** Thrown by placeholders used before activation finished. */ +export const notReady = (what: string): ChordError => new ChordError("not_ready", `${what} is not ready`); diff --git a/packages/chord/src/cluster.ts b/packages/chord/src/cluster.ts new file mode 100644 index 000000000..d9c2157c7 --- /dev/null +++ b/packages/chord/src/cluster.ts @@ -0,0 +1,235 @@ +import type { Connection } from "./connection.ts"; +import { ChordError } from "./errors.ts"; +import type { HostImpl, PluginRef } from "./host.ts"; +import { beginReload, endReload, type Reload, type ReloadOptions, type ReloadReport, reloadError } from "./reload.ts"; +import { type Frame, toWireError } from "./wire.ts"; + +// Cross-host reload, coordinated by the server (the host without a parent): +// server: prepare itself and send `prepare` to every synced child host; collect votes +// all ok: the server installs its new epoch, sends `commit` to every child, and only then starts what may make +// new-epoch calls (tasks, listener and registry callbacks). Links are FIFO, so a child commits before it +// can receive any frame of a root started in the new epoch. +// any failure or missed vote deadline: `abort` everywhere; each host rolls back to where it was. +// Clients (browsers) are not participants: they see shape/reset pushes, or `upgrade_required`. +// A child that connects behind the server's epoch is fenced and reconciled with the same protocol. + +let reloadSeq = 0; + +const modulesFor = (remote: ReloadOptions["remote"], hostName: string | undefined): string[] => { + let best: [string, readonly string[]] | undefined; + for (const entry of Object.entries(remote ?? {})) { + if (hostName?.startsWith(entry[0]) && (best === undefined || entry[0].length > best[0].length)) best = entry; + } + return [...(best?.[1] ?? [])]; +}; + +/** A vote, or its rejection after `deadline` milliseconds. */ +function awaitVote(connection: Connection, id: number, deadline: number): Promise { + let timer: ReturnType | undefined; + const vote = new Promise((resolve, reject) => { + connection.votes.set(id, { resolve, reject }); + timer = setTimeout(() => { + connection.votes.delete(id); + reject(new ChordError("aborted", `votes missing after ${deadline} ms`, { reason: "vote_deadline" })); + }, deadline); + }); + return vote.finally(() => clearTimeout(timer)); +} + +export async function runClusterReload( + host: HostImpl, + plugins: readonly PluginRef[], + options: ReloadOptions, +): Promise { + // Fenced children (behind, being reconciled) are not participants; they reconcile to the new epoch afterwards. + const participants = [...host.connections].filter( + (connection) => + connection.role === "downstream" && connection.peer.kind === "host" && connection.open && connection.synced, + ); + const epoch = host.peekNextEpoch(); + const reload = await beginReload(host, plugins, options, epoch); + const id = ++reloadSeq; + const deadline = options.voteDeadline ?? (options.gateDeadline ?? 5000) + (options.retireDeadline ?? 5000) + 5000; + const votes = participants.map((connection) => { + const name = connection.name ?? "child"; + const vote = awaitVote(connection, id, deadline); + connection.send({ + t: "prepare", + reload: id, + epoch, + plugins: modulesFor(options.remote, name), + options: { + remove: modulesFor(options.remoteRemove, name), + gateDeadline: options.gateDeadline, + retireDeadline: options.retireDeadline, + }, + }); + return vote.then( + (report) => [name, report as ReloadReport | undefined] as const, + (error: unknown) => { + const cause = reloadError(error); + throw new ChordError(cause.code, `${name}: ${cause.message}`, { cause, reason: cause.reason }); + }, + ); + }); + let local = false; + try { + const results = await Promise.allSettled([ + reload.prepare().then(() => { + local = true; + }), + ...votes, + ]); + const failed = results.find((result) => result.status === "rejected"); + if (failed !== undefined) { + for (const connection of participants) { + connection.votes.delete(id); + connection.send({ t: "abort", reload: id }); + } + if (local) await reload.rollback(); + throw failed.reason; + } + const report = reload.commit(() => { + for (const connection of participants) { + connection.send({ t: "commit", reload: id }); + connection.peerEpoch = epoch; + } + }); + host.rememberRemoteModules(options.remote, options.remoteRemove); + const children: Record = {}; + for (const result of results.slice(1)) { + if (result.status !== "fulfilled") continue; + const [name, child] = result.value as readonly [string, ReloadReport | undefined]; + if (child !== undefined) children[name] = child; + } + return { ...report, children }; + } finally { + endReload(host); + } +} + +/** + * Brings a child that connected behind this host's epoch up to it: a reload of the child's modules (the ones + * cluster reloads gave its name) to this host's current epoch, prepared, voted and committed like a cluster reload. + * Until then the child is fenced: nothing is routed to it. The child's old code is never relabeled as a newer epoch. + */ +export async function reconcileChild(host: HostImpl, connection: Connection): Promise { + for (;;) { + const epoch = host.current.id; + if (!connection.open) return; + if (connection.peerEpoch === epoch) { + connection.markSynced(epoch); + return; + } + const id = ++reloadSeq; + const vote = awaitVote(connection, id, 30_000); + connection.send({ + t: "prepare", + reload: id, + epoch, + plugins: modulesFor(host.remoteModules, connection.name), + options: { remove: modulesFor(host.remoteRemoved, connection.name) }, + }); + await vote; + if (host.current.id !== epoch) { + // This host moved on meanwhile: abort and try again with the newer epoch. + connection.send({ t: "abort", reload: id }); + continue; + } + connection.send({ t: "commit", reload: id }); + connection.peerEpoch = epoch; + } +} + +/** One prepare the child received, registered before its first await so an abort or disconnect is never lost. */ +interface ChildOperation { + readonly id: number; + reload: Reload | undefined; + state: "loading" | "preparing" | "prepared"; + cancelled: boolean; +} + +/** + * The child side: prepare and vote, then wait for the server's commit or abort. An abort or a lost parent link can + * arrive at any point: while loading modules, while preparing (pending opens close themselves when they complete), + * or after the vote. The operation is unwound once its pending work settles, and the reload slot is freed. + */ +export class ChildReloads { + readonly #host: HostImpl; + #operation: ChildOperation | undefined; + + constructor(host: HostImpl) { + this.#host = host; + } + + handle(connection: Connection, frame: Extract): void { + if (frame.t === "prepare") { + if (this.#operation !== undefined) { + const error = new ChordError("unavailable", "a reload is already running", { reason: "reload_running" }); + connection.send({ t: "vote", reload: frame.reload, error: toWireError(error) }); + return; + } + const operation: ChildOperation = { id: frame.reload, reload: undefined, state: "loading", cancelled: false }; + this.#operation = operation; + this.#run(operation, connection, frame).catch((error: unknown) => this.#host.report(error)); + return; + } + const operation = this.#operation; + if (operation === undefined || operation.id !== frame.reload) return; + if (frame.t === "abort") { + this.#cancel(operation); + return; + } + if (operation.state !== "prepared") return; + this.#operation = undefined; + try { + operation.reload?.commit(); + } finally { + endReload(this.#host); + } + } + + /** The parent link died: nobody will commit, so roll back (now, or once pending work settles). */ + parentLost(): void { + if (this.#operation !== undefined) this.#cancel(this.#operation); + } + + #cancel(operation: ChildOperation): void { + operation.cancelled = true; + if (operation.state !== "prepared") { + // Loading or preparing: make prepare fail now; #run unwinds when it settles. + operation.reload?.rollback().catch((error: unknown) => this.#host.report(error)); + return; + } + this.#operation = undefined; + operation.reload + ?.rollback() + .catch((error: unknown) => this.#host.report(error)) + .finally(() => endReload(this.#host)); + } + + async #run( + operation: ChildOperation, + connection: Connection, + frame: Extract, + ): Promise { + const cancelled = (): ChordError => + new ChordError("aborted", "reload aborted by the server", { reason: "aborted" }); + try { + operation.reload = await beginReload(this.#host, frame.plugins, frame.options, frame.epoch); + if (operation.cancelled) throw cancelled(); + operation.state = "preparing"; + await operation.reload.prepare(); + if (operation.cancelled) throw cancelled(); + operation.state = "prepared"; + connection.send({ t: "vote", reload: frame.reload, report: operation.reload.summary() }); + } catch (error) { + if (operation.reload !== undefined) { + await operation.reload.rollback().catch((cleanup: unknown) => this.#host.report(cleanup)); + endReload(this.#host); + } + if (this.#operation === operation) this.#operation = undefined; + if (!operation.cancelled) connection.send({ t: "vote", reload: frame.reload, error: toWireError(error) }); + } + } +} diff --git a/packages/chord/src/connection.ts b/packages/chord/src/connection.ts new file mode 100644 index 000000000..5a9907fdd --- /dev/null +++ b/packages/chord/src/connection.ts @@ -0,0 +1,808 @@ +import { contractFromDescription, LOCAL_VARIANT } from "./contract.ts"; +import { type Encoder, encoder } from "./delta/index.ts"; +import { ChordError, invalid } from "./errors.ts"; +import type { HostImpl } from "./host.ts"; +import type { Link } from "./link.ts"; +import { type CtxImpl, HandleImpl, kindOf } from "./record.ts"; +import type { RemoteChannel, RemoteHandle, StateSink } from "./remote.ts"; +import { assertValid } from "./schema.ts"; +import type { Ctx, Principal, Readable, StateEvent } from "./types.ts"; +import { type Frame, fromWireError, isFrame, type RootRef, toWireError } from "./wire.ts"; + +export interface LinkOptions { + /** Milliseconds between pings (default 15000). */ + readonly pingInterval?: number; + /** Close the link when nothing arrived for this long (default 45000). Everything held through it is released. */ + readonly pingTimeout?: number; +} + +export interface AcceptOptions extends LinkOptions { + /** The principal of whatever connects, from the app's authenticator. Every request on the link acts as it. */ + readonly principal: Principal; + /** + * What the peer is, decided here and never by its `hello`: `client` (default; a browser: no roots, no epochs, not + * a reload participant, not addressable) or `host` (a child host; needs `name`). + */ + readonly kind?: "host" | "client"; + /** A child host's name: its hello must match, and it is reachable `at` this name. */ + readonly name?: string; + /** On `invalidatePrincipal`: a new principal keeps the link, `undefined` closes it (default: close). */ + readonly restamp?: (principal: Principal) => Principal | undefined; +} + +export interface Peer { + readonly kind: "host" | "client"; + /** Configured (downstream hosts) or announced by the trusted parent (upstream). */ + readonly name: string | undefined; +} + +/** A hold the peer made on this host: served locally or relayed to another link. */ +type Served = + | { readonly kind: "local"; readonly handle: HandleImpl; readonly principal: Principal; readonly feeds: Set } + | { readonly kind: "relay"; readonly relay: RelayHold }; + +type Pending = { readonly resolve: (value: unknown) => void; readonly reject: (error: unknown) => void }; + +const unavailable = (message: string): ChordError => new ChordError("unavailable", message); + +// ─── relay ─────────────────────────────────────────────────────────────────── + +/** + * A hold relayed from an origin link to a downstream link. It owns everything done through it: subscriptions and + * calls queued while the route resolves, and the downstream twins once forwarded. `close` undoes all of it exactly + * once (cancels calls downstream, drops subscriptions, releases the downstream hold, rejects waiters). + */ +export class RelayHold { + readonly origin: Connection; + readonly frame: Extract; + readonly principal: Principal; + readonly client: boolean; + /** The origin's authorization generation when the hold arrived; a revocation bumps it. */ + readonly generation: number; + down: Connection | undefined; + downId = 0; + closed = false; + readonly calls = new Map(); + readonly subs = new Map(); + readonly #waiters = new Set(); + + constructor(origin: Connection, frame: Extract, principal: Principal, client: boolean) { + this.origin = origin; + this.frame = frame; + this.principal = principal; + this.client = client; + this.generation = origin.generation; + } + + get originId(): number { + return this.frame.id; + } + + /** Still authorized as it was when it arrived. */ + get current(): boolean { + return !this.closed && this.origin.open && this.origin.generation === this.generation; + } + + /** The downstream connection once the route resolved; rejects if the relay closes first. */ + ready(): Promise { + if (this.closed) return Promise.reject(new ChordError("gone", "relayed hold closed")); + if (this.down !== undefined) return Promise.resolve(this.down); + return new Promise((resolve, reject) => { + this.#waiters.add({ resolve: resolve as (value: unknown) => void, reject }); + }); + } + + /** Forwards the hold (and what queued meanwhile) to `down`. */ + open(down: Connection): void { + if (this.closed) return; + this.down = down; + this.downId = down.nextId(); + down.relayedHolds.set(this.downId, this); + down.send({ + ...this.frame, + id: this.downId, + principal: this.principal, + ...(this.client ? { client: true } : {}), + }); + for (const [originSubId, entry] of this.subs) + if (entry.downId === undefined) this.#forwardSub(originSubId, entry); + for (const waiter of this.#waiters) waiter.resolve(down); + this.#waiters.clear(); + } + + subscribe(originSubId: number, member: string): void { + const entry = { member, downId: undefined as number | undefined }; + this.subs.set(originSubId, entry); + this.origin.relaySubs.set(originSubId, this); + if (this.down !== undefined) this.#forwardSub(originSubId, entry); + } + + #forwardSub(originSubId: number, entry: { readonly member: string; downId: number | undefined }): void { + const down = this.down!; + entry.downId = down.nextId(); + down.relayedSubs.set(entry.downId, { relay: this, originSubId }); + down.send({ t: "sub", id: entry.downId, hold: this.downId, member: entry.member }); + } + + unsubscribe(originSubId: number, forward = true): void { + const entry = this.subs.get(originSubId); + if (entry === undefined) return; + this.subs.delete(originSubId); + this.origin.relaySubs.delete(originSubId); + if (entry.downId !== undefined && this.down !== undefined) { + this.down.relayedSubs.delete(entry.downId); + if (forward) this.down.send({ t: "unsub", id: entry.downId }); + } + } + + /** + * Closes the relay. `releaseDown`: the downstream hold is still there (origin released, disconnected, or was + * revoked): cancel its calls and release it. `notify`: tell the origin the hold is gone. + */ + close(error: unknown, options: { readonly releaseDown: boolean; readonly notify: boolean }): void { + if (this.closed) return; + this.closed = true; + for (const call of [...this.calls.values()]) call.cancel(error, options.releaseDown); + for (const originSubId of [...this.subs.keys()]) this.unsubscribe(originSubId, options.releaseDown); + if (this.origin.served.get(this.originId)?.kind === "relay") this.origin.served.delete(this.originId); + if (this.down !== undefined) { + this.down.relayedHolds.delete(this.downId); + if (options.releaseDown) this.down.send({ t: "release", id: this.downId }); + } + for (const waiter of this.#waiters) waiter.reject(error); + this.#waiters.clear(); + if (options.notify) this.origin.send({ t: "gone", id: this.originId, error: toWireError(error) }); + } +} + +/** One relayed call: settles once, by result, cancellation or the relay closing; removes both mappings. */ +export class RelayCall { + readonly relay: RelayHold; + readonly originId: number; + readonly downId: number; + readonly #pending: Pending; + #settled = false; + + constructor(relay: RelayHold, originId: number, downId: number, pending: Pending) { + this.relay = relay; + this.originId = originId; + this.downId = downId; + this.#pending = pending; + relay.calls.set(originId, this); + relay.down?.relayedCalls.set(downId, this); + } + + finish(error: unknown, value?: unknown): void { + if (this.#settled) return; + this.#settled = true; + this.relay.calls.delete(this.originId); + this.relay.down?.relayedCalls.delete(this.downId); + if (error === undefined) this.#pending.resolve(value); + else this.#pending.reject(error); + } + + /** Cancels downstream (when the downstream hold still exists) and settles with `error`. */ + cancel(error: unknown, forward: boolean): void { + if (this.#settled) return; + if (forward) this.relay.down?.send({ t: "cancel", id: this.downId }); + this.finish(error); + } +} + +// ─── connection ────────────────────────────────────────────────────────────── + +/** One end of a link on a host: consumer of remote handles, provider for the peer's holds, and relay. */ +export class Connection implements RemoteChannel { + readonly host: HostImpl; + readonly link: Link; + /** `upstream`: the link to this host's parent. `downstream`: a link this host accepted. */ + readonly role: "upstream" | "downstream"; + readonly peer: Peer; + principal: Principal; + readonly restamp: ((principal: Principal) => Principal | undefined) | undefined; + /** Bumped when the stamped principal is revoked or re-stamped: relays from before are refused. */ + generation = 0; + /** The epoch a downstream host peer runs (from its hello, then from commits). */ + peerEpoch: number | undefined; + /** A downstream host peer is usable once it runs this host's current epoch (see `HostImpl.peerHello`). */ + synced: boolean; + /** Epochs this child announced while behind this host (fenced): this host never ran them. */ + readonly fencedEpochs = new Set(); + /** Roots of fenced epochs, by the epoch of this host they were admitted in. */ + readonly #admitted = new Map(); + #parentName: string | undefined; + readonly #syncWaiters = new Set(); + #nextId = 1; + #lastSeen = Date.now(); + #pinger: ReturnType | undefined; + // consumer side: what this host asked of the peer + readonly remotes = new Map(); + readonly calls = new Map(); + readonly subs = new Map void>(); + // provider side: what the peer asked of this host + readonly served = new Map(); + readonly incoming = new Map(); + readonly feeds = new Map void>(); + /** Origin side of relays: the peer's sub ids that are relayed. */ + readonly relaySubs = new Map(); + // downstream side of relays: ids this host used on this link + readonly relayedHolds = new Map(); + readonly relayedCalls = new Map(); + readonly relayedSubs = new Map(); + /** Reload votes awaited from this (child) peer. */ + readonly votes = new Map(); + + constructor( + host: HostImpl, + link: Link, + config: { + readonly role: "upstream" | "downstream"; + readonly principal: Principal; + readonly peer: Peer; + readonly restamp?: (principal: Principal) => Principal | undefined; + } & LinkOptions, + ) { + this.host = host; + this.link = link; + this.role = config.role; + this.peer = config.peer; + this.principal = config.principal; + this.restamp = config.restamp; + this.synced = !(this.role === "downstream" && this.peer.kind === "host"); + link.onClose((reason) => this.#closed(reason)); + link.receive((frame) => this.#receive(frame)); + this.send({ t: "hello", name: host.name, kind: host.kind, epoch: host.current.id }); + const interval = config.pingInterval ?? 15_000; + const timeout = config.pingTimeout ?? 45_000; + let n = 0; + this.#pinger = setInterval(() => { + if (Date.now() - this.#lastSeen > timeout) link.close("ping timeout"); + else this.send({ t: "ping", n: ++n }); + }, interval); + this.#pinger.unref?.(); + } + + get open(): boolean { + return !this.link.closed; + } + + /** The peer's name: configured for downstream hosts, announced by the parent upstream. */ + get name(): string | undefined { + return this.role === "upstream" ? this.#parentName : this.peer.name; + } + + send(frame: Frame): void { + this.link.send(frame); + } + + nextId(): number { + return this.#nextId++; + } + + /** Resolves once the peer is usable (a host peer runs this host's epoch); rejects if the link closes first. */ + whenSynced(): Promise { + if (!this.open) return Promise.reject(unavailable(`link to ${this.name ?? "peer"} closed`)); + if (this.synced) return Promise.resolve(); + return new Promise((resolve, reject) => this.#syncWaiters.add({ resolve: resolve as () => void, reject })); + } + + markSynced(epoch: number): void { + this.peerEpoch = epoch; + this.synced = true; + for (const waiter of this.#syncWaiters) waiter.resolve(undefined); + this.#syncWaiters.clear(); + } + + /** Revokes the stamped principal: relays made under it close at once (also those still resolving a route). */ + revoke(next: Principal | undefined): void { + this.generation++; + const error = new ChordError("gone", "principal invalidated"); + for (const served of [...this.served.values()]) { + if (served.kind === "relay") served.relay.close(error, { releaseDown: true, notify: true }); + } + if (next === undefined) this.link.close("principal invalidated"); + else this.principal = next; + } + + /** A protocol violation: the peer cannot be trusted with anything on this link any more. */ + #violation(reason: string): void { + this.host.report(new ChordError("invalid", `link to ${this.name ?? "peer"}: ${reason}`, { reason: "protocol" })); + this.link.close(`protocol error: ${reason}`); + } + + // ─── consumer side (RemoteChannel) ─── + + hold(handle: RemoteHandle): number { + const id = this.nextId(); + this.remotes.set(id, handle); + handle.bound(this, id); + const principal = handle.principal; + this.send({ + t: "hold", + id, + at: handle.at, + token: handle.contract.description, + input: handle.input, + ...(principal === undefined ? {} : { principal }), + ...(handle.owner === "client" ? { client: true } : {}), + }); + return id; + } + + release(id: number): void { + this.remotes.delete(id); + this.send({ t: "release", id }); + } + + call(hold: number, member: string, args: unknown, root: RootRef | undefined, signal: AbortSignal): Promise { + const id = this.nextId(); + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new ChordError("aborted", "call aborted")); + return; + } + const onAbort = (): void => { + this.calls.delete(id); + this.send({ t: "cancel", id }); + reject(new ChordError("aborted", "call aborted", { cause: signal.reason })); + }; + signal.addEventListener("abort", onAbort, { once: true }); + const done = (): void => signal.removeEventListener("abort", onAbort); + this.calls.set(id, { + resolve: (value) => { + done(); + resolve(value); + }, + reject: (error) => { + done(); + reject(error); + }, + }); + // Clients never carry roots or epochs: the server starts roots for them. + const carried = root !== undefined && this.host.kind === "host"; + if (carried) this.host.rootSent(root.rootId, this); + this.send({ t: "call", id, hold, member, args, ...(carried ? { root } : {}) }); + }); + } + + sub(hold: number, member: string, sink: (frame: StateSink) => void): number { + const id = this.nextId(); + this.subs.set(id, sink); + this.send({ t: "sub", id, hold, member }); + return id; + } + + unsub(id: number): void { + if (this.subs.delete(id)) this.send({ t: "unsub", id }); + } + + // ─── receiving ─── + + #receive(frame: unknown): void { + this.#lastSeen = Date.now(); + if (!isFrame(frame)) { + this.#violation("malformed frame"); + return; + } + try { + this.#dispatch(frame); + } catch (error) { + this.host.report(error); + } + } + + #dispatch(frame: Frame): void { + switch (frame.t) { + case "hello": + this.#onHello(frame); + return; + case "ping": + this.send({ t: "pong", n: frame.n }); + return; + case "pong": + case "bye": + return; + case "hold": + this.#onHold(frame); + return; + case "release": + this.#dropServed(frame.id, true); + return; + case "call": + this.#onCall(frame); + return; + case "cancel": + this.incoming.get(frame.id)?.abort(new ChordError("aborted", "cancelled by caller")); + return; + case "sub": + this.#onSub(frame); + return; + case "unsub": + this.#onUnsub(frame.id); + return; + case "shape": + case "gone": + this.#onHoldReply(frame); + return; + case "result": + this.#onResult(frame); + return; + case "state": + this.#onState(frame); + return; + case "root_end": + // Honored only from the link the root came over (see HostImpl.foreignRootEnded). + this.#admitted.delete(frame.rootId); + this.host.foreignRootEnded(frame.rootId, this); + return; + case "invalidate": + case "prepare": + case "commit": + case "abort": + case "retire": + // Control frames only from the parent. + if (this.role !== "upstream") { + this.#violation(`${frame.t} from a child`); + return; + } + if (frame.t === "invalidate") { + this.host.invalidatePrincipal(frame.pattern).catch((error: unknown) => this.host.report(error)); + } else if (frame.t === "retire") this.host.allowRetire(frame.epoch); + else this.host.onReloadFrame(this, frame); + return; + case "vote": { + if (this.role !== "downstream" || this.peer.kind !== "host") { + this.#violation("vote from a non-participant"); + return; + } + const vote = this.votes.get(frame.reload); + this.votes.delete(frame.reload); + if (frame.error === undefined) vote?.resolve(frame.report); + else vote?.reject(fromWireError(frame.error)); + return; + } + } + } + + /** + * The peer's hello must agree with what this side decided about it. A downstream host announces its epoch: the + * host either finds it in step or fences and reconciles it (never renumbers). + */ + #onHello(frame: Extract): void { + if (this.role === "upstream") { + if (frame.kind !== "host") { + this.#violation("parent is not a host"); + return; + } + this.#parentName ??= frame.name; + return; + } + if (frame.kind !== this.peer.kind || (this.peer.name !== undefined && frame.name !== this.peer.name)) { + this.#violation( + `hello ${frame.kind} ${frame.name} does not match the accepted ${this.peer.kind} ${this.peer.name ?? ""}`, + ); + return; + } + if (this.peer.kind === "host") this.host.peerHello(this, frame.epoch); + } + + /** + * Principal stamping: a downstream link acts as the principal it was accepted with, whatever the frame says. Only + * the parent may forward a principal (relayed requests); its own requests act as `{ host: parentName }`. + */ + #principalOf(frame: { readonly principal?: Principal }): Principal { + if (this.role === "downstream") return this.principal; + return frame.principal ?? { host: this.#parentName ?? "parent" }; + } + + #isClient(frame: { readonly client?: boolean }): boolean { + return this.role === "downstream" ? this.peer.kind === "client" : frame.client === true; + } + + #onHold(frame: Extract): void { + if (this.served.has(frame.id)) { + this.#violation(`duplicate hold id ${frame.id}`); + return; + } + const principal = this.#principalOf(frame); + const client = this.#isClient(frame); + if (frame.at !== this.host.name) { + if (this.role === "upstream") { + this.send({ t: "gone", id: frame.id, error: toWireError(unavailable(`${this.host.name} does not relay`)) }); + return; + } + const relay = new RelayHold(this, frame, principal, client); + this.served.set(frame.id, { kind: "relay", relay }); + this.host.relay(relay).catch((error: unknown) => relay.close(error, { releaseDown: true, notify: true })); + return; + } + let handle: HandleImpl; + try { + const token = contractFromDescription(frame.token); + if (token.scope !== "remote") throw invalid("token", `${token.name} is local; only remote types cross links`); + const acquired = this.host.acquire(token, frame.input, { principal }, client ? "client" : "connection"); + if (!(acquired instanceof HandleImpl)) throw unavailable("not a local handle"); + handle = acquired; + } catch (error) { + this.send({ t: "gone", id: frame.id, error: toWireError(client ? clientError(error) : error) }); + return; + } + const served: Served = { kind: "local", handle, principal, feeds: new Set() }; + this.served.set(frame.id, served); + const shape = (): void => { + const current = handle.shape; + if (current === undefined || this.served.get(frame.id) !== served) return; + this.send({ + t: "shape", + id: frame.id, + kind: current.kind, + members: current.members, + contract: current.contract, + epoch: current.epoch, + }); + }; + handle.onShape(shape); + handle.goneListeners.add((error) => { + if (this.served.get(frame.id) !== served) return; + this.#dropServed(frame.id, false); + this.send({ t: "gone", id: frame.id, error: toWireError(error) }); + }); + if (handle.status === "live") shape(); + } + + /** Forgets a served hold: unsubscribes its feeds and releases it (locally or downstream). */ + #dropServed(id: number, release: boolean): void { + const served = this.served.get(id); + if (served === undefined) return; + this.served.delete(id); + if (served.kind === "relay") { + served.relay.close(new ChordError("released", "relayed hold released"), { + releaseDown: release, + notify: false, + }); + return; + } + for (const feed of served.feeds) this.#onUnsub(feed); + if (release) served.handle.release(); + } + + #onCall(frame: Extract): void { + if (this.incoming.has(frame.id)) { + this.#violation(`duplicate call id ${frame.id}`); + return; + } + const served = this.served.get(frame.hold); + if (served === undefined) { + this.send({ t: "result", id: frame.id, error: toWireError(new ChordError("gone", "unknown hold")) }); + return; + } + const controller = new AbortController(); + this.incoming.set(frame.id, controller); + // Roots are honored from hosts only; a request without one (or from a client) starts one here. + const root = this.peer.kind === "host" ? this.#admitRoot(frame.root) : undefined; + const principal = served.kind === "local" ? served.principal : served.relay.principal; + const options = { principal, signal: controller.signal }; + const run = + served.kind === "local" + ? (ctx: Ctx) => served.handle.record.call(served.handle, frame.member, frame.args, ctx as CtxImpl) + : (ctx: Ctx) => + relayCall( + served.relay, + frame, + { rootId: ctx.rootId, epoch: ctx.epoch }, + controller.signal, + this.host, + ); + const result = + root === undefined + ? this.host.root(run, options) + : this.host.joinRoot(root.rootId, root.epoch, run, { ...options, from: this }); + result + .then( + (value) => this.send({ t: "result", id: frame.id, value }), + (error: unknown) => this.send({ t: "result", id: frame.id, error: toWireError(error) }), + ) + .finally(() => this.incoming.delete(frame.id)); + } + + /** + * A child that connected behind this host runs roots of an epoch this host never had (it was fenced and is being, + * or was, reconciled). Such a root is admitted in this host's current epoch at its first call here, and keeps that + * epoch for its later calls (`superseded` once this host retired it). Other unknown epochs stay `superseded`. + */ + #admitRoot(root: RootRef | undefined): RootRef | undefined { + if (root === undefined || this.role !== "downstream" || !this.fencedEpochs.has(root.epoch)) return root; + if (this.host.hasEpoch(root.epoch)) return root; + let epoch = this.#admitted.get(root.rootId); + if (epoch === undefined) { + epoch = this.host.current.id; + this.#admitted.set(root.rootId, epoch); + } + return { rootId: root.rootId, epoch }; + } + + #onSub(frame: Extract): void { + if (this.feeds.has(frame.id) || this.relaySubs.has(frame.id)) { + this.#violation(`duplicate sub id ${frame.id}`); + return; + } + const served = this.served.get(frame.hold); + if (served === undefined) { + this.send({ t: "state", id: frame.id, error: toWireError(new ChordError("gone", "unknown hold")) }); + return; + } + if (served.kind === "relay") { + served.relay.subscribe(frame.id, frame.member); + return; + } + const readable = served.handle.member(frame.member) as Readable | undefined; + if (readable === undefined || typeof readable.events !== "function") { + this.send({ + t: "state", + id: frame.id, + error: toWireError(new ChordError("not_in_variant", `${frame.member} is not state`)), + }); + return; + } + let stream: Encoder = encoder(); + const send = (event: StateEvent): void => { + if (event.type === "ops") { + this.send({ t: "state", id: frame.id, ops: stream.encode(event.ops) }); + return; + } + // A loading provider handle has nothing yet: the hydrated value follows as a reset. + if (event.value === undefined) return; + // Snapshots are validated against the effective schema once per (re)subscription; ops only in dev mode. + try { + validateSnapshot(served.handle, frame.member, event.value); + } catch (error) { + this.send({ t: "state", id: frame.id, error: toWireError(error) }); + return; + } + stream = encoder(); + this.send( + event.type === "snap" + ? { t: "state", id: frame.id, snap: event.value } + : { t: "state", id: frame.id, reset: event.value }, + ); + }; + const unsubscribe = readable.events(send); + this.feeds.set(frame.id, unsubscribe); + served.feeds.add(frame.id); + } + + #onUnsub(id: number): void { + const unsubscribe = this.feeds.get(id); + if (unsubscribe !== undefined) { + this.feeds.delete(id); + unsubscribe(); + } + this.relaySubs.get(id)?.unsubscribe(id); + } + + #onHoldReply(frame: Extract): void { + const relay = this.relayedHolds.get(frame.id); + if (relay !== undefined) { + if (frame.t === "gone") relay.close(fromWireError(frame.error), { releaseDown: false, notify: true }); + else if (!relay.closed) relay.origin.send({ ...frame, id: relay.originId }); + return; + } + const handle = this.remotes.get(frame.id); + if (handle === undefined) return; + if (frame.t === "gone") { + this.remotes.delete(frame.id); + handle.gone(fromWireError(frame.error)); + return; + } + handle.shaped({ kind: frame.kind, members: frame.members, contract: frame.contract, epoch: frame.epoch }); + } + + #onResult(frame: Extract): void { + const relayed = this.relayedCalls.get(frame.id); + if (relayed !== undefined) { + relayed.finish(frame.error === undefined ? undefined : fromWireError(frame.error), frame.value); + return; + } + const pending = this.calls.get(frame.id); + this.calls.delete(frame.id); + if (frame.error === undefined) pending?.resolve(frame.value); + else pending?.reject(fromWireError(frame.error)); + } + + #onState(frame: Extract): void { + const relayed = this.relayedSubs.get(frame.id); + if (relayed !== undefined) { + relayed.relay.origin.send({ ...frame, id: relayed.originSubId }); + return; + } + this.subs.get(frame.id)?.(frame); + } + + // ─── lifetime ─── + + #closed(reason: string): void { + if (this.#pinger !== undefined) clearInterval(this.#pinger); + const error = unavailable(`link to ${this.name ?? "peer"} closed: ${reason}`); + // As origin: everything the peer held here, relayed or not, is released (downstream calls cancelled). + for (const id of [...this.served.keys()]) this.#dropServed(id, true); + for (const controller of this.incoming.values()) controller.abort(error); + this.incoming.clear(); + for (const pending of this.calls.values()) pending.reject(error); + this.calls.clear(); + this.subs.clear(); + for (const vote of this.votes.values()) vote.reject(error); + this.votes.clear(); + for (const waiter of this.#syncWaiters) waiter.reject(error); + this.#syncWaiters.clear(); + // As downstream of relays: their origins lose the hold. + for (const relay of [...this.relayedHolds.values()]) relay.close(error, { releaseDown: false, notify: true }); + this.relayedHolds.clear(); + this.relayedCalls.clear(); + this.relayedSubs.clear(); + for (const handle of this.remotes.values()) { + // An upstream link may be replaced (reconnect); a downstream host that went away is gone. + if (this.role === "upstream") handle.detached(); + else handle.gone(error); + } + this.remotes.clear(); + this.host.connectionClosed(this); + } +} + +/** + * A call through a relay: waits for the route, then forwards with the root it runs in. It settles exactly once; a + * cancel (or the origin going away) is forwarded downstream and removes both mappings. + */ +async function relayCall( + relay: RelayHold, + frame: Extract, + root: RootRef, + signal: AbortSignal, + host: HostImpl, +): Promise { + const abortError = (): ChordError => new ChordError("aborted", "call cancelled", { cause: signal.reason }); + if (signal.aborted) throw abortError(); + let onAbort = (): void => {}; + const aborted = new Promise((_, reject) => { + onAbort = () => reject(abortError()); + signal.addEventListener("abort", onAbort, { once: true }); + }); + aborted.catch(() => {}); + try { + const down = await Promise.race([relay.ready(), aborted]); + if (relay.closed) throw new ChordError("gone", "relayed hold closed"); + return await new Promise((resolve, reject) => { + const call = new RelayCall(relay, frame.id, down.nextId(), { resolve, reject }); + signal.addEventListener("abort", () => call.cancel(abortError(), true), { once: true }); + host.rootSent(root.rootId, down); + down.send({ t: "call", id: call.downId, hold: relay.downId, member: frame.member, args: frame.args, root }); + }); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + +/** + * Validates a snapshot against the schema the consumer is promised: the provider's own schema for a direct binding, + * the consumer's contract for one through an `accepts` adapter (the adapter's view is what the consumer gets). + */ +function validateSnapshot(handle: HandleImpl, member: string, value: unknown): void { + if (value === undefined) return; + const installed = handle.record.currentInstalled(); + if (installed === undefined) return; + const view = handle.currentView(); + const kind = view === undefined ? kindOf(installed.behavior) : kindOf(view); + const description = + view !== undefined && view !== installed.behavior + ? handle.contract.description.variants[kind ?? LOCAL_VARIANT]?.[member] + : installed.service.contract.description.variants[kindOf(installed.behavior) ?? LOCAL_VARIANT]?.[member]; + if (description?.kind === "state") assertValid("state", `${handle.address}.${member}`, description.schema, value); +} + +/** Clients (browsers) holding an outdated contract are told to reload their code. */ +function clientError(error: unknown): unknown { + if (error instanceof ChordError && error.code === "contract_mismatch") { + return new ChordError("upgrade_required", error.message, { cause: error }); + } + return error; +} diff --git a/packages/chord/src/context/index.ts b/packages/chord/src/context/index.ts index fbe54bcc4..7426083f6 100644 --- a/packages/chord/src/context/index.ts +++ b/packages/chord/src/context/index.ts @@ -1,4 +1,16 @@ -import type { Context, ContextKey } from "../types.ts"; +/** Typed identity for one value carried by a {@link Context}. */ +export interface ContextKey { + readonly token: symbol; + /** Type-only marker that prevents keys with different value types from being interchangeable. */ + readonly valueType?: (value: T) => T; +} + +/** Immutable invocation-scoped values passed explicitly through operations. */ +export interface Context { + readonly abortSignal: AbortSignal | undefined; + value(key: ContextKey): T | undefined; + toString(): string; +} const ABORT_SIGNAL_CONTEXT_KEY: ContextKey = Object.freeze({ token: Symbol("chord.abortSignal"), diff --git a/packages/chord/src/contract.ts b/packages/chord/src/contract.ts new file mode 100644 index 000000000..cf918cea7 --- /dev/null +++ b/packages/chord/src/contract.ts @@ -0,0 +1,289 @@ +import type { Static, TSchema } from "typebox"; +import { ChordError } from "./errors.ts"; +import { canonicalJson, fits, hash64 } from "./schema.ts"; +import type { Ctx, Exposed, HandleBase, Readable, StateSource } from "./types.ts"; + +const MEMBER = Symbol("chord.member"); + +export interface MethodSpec { + readonly [MEMBER]: "method"; + readonly args: A; + readonly result: R; +} + +export interface OpaqueSpec { + readonly [MEMBER]: "opaque"; + readonly id: string; + /** Type carrier only; never set. */ + readonly type?: T; +} + +export const method = (args: A, result: R): MethodSpec => ({ + [MEMBER]: "method", + args, + result, +}); + +/** A native in-process object identified by `name@version` (local contracts only). */ +export const opaque = (id: string): OpaqueSpec => { + if (!/^[^@\s]+@\d+$/.test(id)) throw new TypeError(`opaque id must be name@version: ${id}`); + return { [MEMBER]: "opaque", id }; +}; + +export interface VariantDef { + readonly state?: Readonly>; + readonly methods?: Readonly>; +} + +export interface RemoteDef { + readonly version: number; + readonly input: TSchema; + readonly variants: Readonly>; +} + +/** A local member: a method, an opaque native object, or a state member described by its schema. */ +export type LocalMember = MethodSpec | OpaqueSpec | TSchema; + +export interface LocalDef { + readonly version: number; + readonly input: TSchema; + readonly members?: Readonly>; +} + +export type MemberDescription = + | { readonly kind: "method"; readonly args: TSchema; readonly result: TSchema } + | { readonly kind: "state"; readonly schema: TSchema } + | { readonly kind: "opaque"; readonly id: string }; + +/** Local contracts have one unnamed variant. */ +export const LOCAL_VARIANT = ""; + +export interface Description { + readonly name: string; + readonly scope: "remote" | "local"; + readonly version: number; + readonly input: TSchema; + readonly variants: Readonly>>>; +} + +interface ContractBase { + readonly name: string; + readonly version: number; + readonly input: TSchema; + readonly description: Description; + readonly fingerprint: string; +} + +export interface RemoteContract extends ContractBase { + readonly scope: "remote"; + readonly def: D; +} + +export interface LocalContract extends ContractBase { + readonly scope: "local"; + readonly def: D; +} + +export type AnyContract = RemoteContract | LocalContract; + +/** Names a handle uses itself; members may not use them. */ +export const RESERVED_MEMBERS: ReadonlySet = new Set([ + "kind", + "status", + "ready", + "shape", + "error", + "onShape", + "onStatus", + "release", + "then", + "contract", +]); + +function describeMember(name: string, member: unknown, scope: "remote" | "local"): MemberDescription { + if (RESERVED_MEMBERS.has(name)) throw new TypeError(`member name is reserved: ${name}`); + const brand = (member as { [MEMBER]?: string })[MEMBER]; + if (brand === "method") { + const spec = member as MethodSpec; + return { kind: "method", args: spec.args, result: spec.result }; + } + if (brand === "opaque") { + if (scope === "remote") throw new TypeError(`remote contracts cannot expose opaque members: ${name}`); + return { kind: "opaque", id: (member as OpaqueSpec).id }; + } + return { kind: "state", schema: member as TSchema }; +} + +function finish(contract: Omit): C { + return { ...contract, fingerprint: hash64(canonicalJson(contract.description)) } as C; +} + +function checkHeader(name: string, version: number): void { + if (!name) throw new TypeError("contract name is empty"); + if (!Number.isInteger(version) || version < 1) throw new TypeError(`contract version must be a positive integer`); +} + +export function remote(name: string, def: D): RemoteContract { + checkHeader(name, def.version); + const variants: Record> = {}; + if (Object.keys(def.variants).length === 0) throw new TypeError(`remote contract ${name} has no variants`); + for (const [kind, variant] of Object.entries(def.variants)) { + const members: Record = {}; + for (const [key, schema] of Object.entries(variant.state ?? {})) + members[key] = describeMember(key, schema, "remote"); + for (const [key, spec] of Object.entries(variant.methods ?? {})) { + if (key in members) throw new TypeError(`member ${key} is both state and method in ${name}.${kind}`); + members[key] = describeMember(key, spec, "remote"); + if (members[key].kind !== "method") throw new TypeError(`methods.${key} is not a method()`); + } + variants[kind] = members; + } + const description: Description = { name, scope: "remote", version: def.version, input: def.input, variants }; + return finish>({ + scope: "remote", + name, + version: def.version, + input: def.input, + def, + description, + }); +} + +export function local(name: string, def: D): LocalContract { + checkHeader(name, def.version); + const members: Record = {}; + for (const [key, member] of Object.entries(def.members ?? {})) members[key] = describeMember(key, member, "local"); + const description: Description = { + name, + scope: "local", + version: def.version, + input: def.input, + variants: { [LOCAL_VARIANT]: members }, + }; + return finish>({ scope: "local", name, version: def.version, input: def.input, def, description }); +} + +/** + * A consumer token received over a link (only its description travels). Has no `def`, so it is only good for binding + * and validation, never for TypeScript types. + */ +export function contractFromDescription(description: Description): AnyContract { + const { name, scope, version, input } = description; + if (typeof name !== "string" || (scope !== "remote" && scope !== "local") || !Number.isInteger(version)) { + throw new ChordError("invalid", "malformed contract token", { reason: "token" }); + } + const base = { name, version, input, description, fingerprint: hash64(canonicalJson(description)) }; + return scope === "remote" + ? { ...base, scope, def: { version, input, variants: {} } } + : { ...base, scope, def: { version, input } }; +} + +/** Returns why `provider` does not cover `consumer`, or undefined when it does. */ +export function coverage(provider: Description, consumer: Description): string | undefined { + if (provider.name !== consumer.name || provider.scope !== consumer.scope) return "different type"; + if (!fits(consumer.input, provider.input)) return "input"; + for (const [kind, consumerMembers] of Object.entries(consumer.variants)) { + const providerMembers = provider.variants[kind]; + if (providerMembers === undefined) return `variant ${kind} missing`; + for (const [name, wanted] of Object.entries(consumerMembers)) { + const offered = providerMembers[name]; + if (offered === undefined || offered.kind !== wanted.kind) return `${kind}.${name} missing or of another kind`; + if (offered.kind === "method" && wanted.kind === "method") { + if (!fits(wanted.args, offered.args)) return `${kind}.${name} arguments`; + if (!fits(offered.result, wanted.result)) return `${kind}.${name} result`; + } else if (offered.kind === "state" && wanted.kind === "state") { + if (!fits(offered.schema, wanted.schema)) return `${kind}.${name} state`; + } else if (offered.kind === "opaque" && wanted.kind === "opaque" && offered.id !== wanted.id) { + return `${kind}.${name} opaque ${offered.id} != ${wanted.id}`; + } + } + } + return undefined; +} + +/** Maps the provider's current behavior to the view a consumer of an older contract version expects. */ +export type Adapter = (current: Record) => Record; +export type Accepts = Readonly>; + +/** How a consumer token binds to a provider: directly or through an `accepts` adapter. Throws contract_mismatch. */ +export function bind(provider: AnyContract, accepts: Accepts | undefined, consumer: AnyContract): Adapter | undefined { + if (provider.fingerprint === consumer.fingerprint) return undefined; + if (provider.version === consumer.version) { + const reason = coverage(provider.description, consumer.description); + if (reason === undefined) return undefined; + throw new ChordError( + "contract_mismatch", + `${provider.name}@${provider.version} does not cover the consumer's description (${reason}); forgotten version bump?`, + ); + } + const adapter = accepts?.[consumer.version]; + if (adapter === undefined) { + throw new ChordError( + "contract_mismatch", + `${provider.name}: provider version ${provider.version}, consumer version ${consumer.version}, no accepts adapter`, + ); + } + return adapter; +} + +// ─── Static types ──────────────────────────────────────────────────────────── + +type UnionToIntersection = (U extends unknown ? (value: U) => void : never) extends (value: infer I) => void + ? I + : never; +type Empty = Record; +type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; +type StaticOf = S extends TSchema ? Static : never; +type MethodsOf = V extends { readonly methods: infer M extends Readonly> } ? M : Empty; + +export type Implementation = M extends MethodSpec + ? (args: Static, ctx: Ctx) => Static | Promise> + : never; +export type Call = M extends MethodSpec + ? (args: Static, ctx?: Ctx) => Promise> + : never; + +type VariantBehavior = { readonly kind: K } & { + readonly [S in keyof StateOf]: StateSource[S]>>; +} & { + readonly [M in keyof MethodsOf]: Implementation[M]>; +}; +type VariantHandle = { readonly [S in keyof StateOf]: Readable[S]> | undefined> } & { + readonly [M in keyof MethodsOf]: Call[M]>; +}; + +type LocalBehaviorMember = M extends MethodSpec + ? Implementation + : M extends OpaqueSpec + ? T | Exposed + : M extends TSchema + ? StateSource> + : never; +type LocalHandleMember = M extends MethodSpec + ? Call + : M extends OpaqueSpec + ? T | undefined + : M extends TSchema + ? Readable | undefined> + : never; +type LocalMembers = D extends { readonly members: infer M extends Readonly> } + ? M + : Empty; + +export type InputOf = C extends { readonly def: { readonly input: infer I extends TSchema } } ? Static : never; + +/** What a factory returns for contract C. */ +export type BehaviorOf = C extends RemoteContract + ? { [K in keyof D["variants"]]: VariantBehavior }[keyof D["variants"]] + : C extends LocalContract + ? { readonly [K in keyof LocalMembers]: LocalBehaviorMember[K]> } & Record + : never; + +/** Late-bound members a handle exposes for contract C (all variants merged). */ +export type MembersOf = C extends RemoteContract + ? UnionToIntersection<{ [K in keyof D["variants"]]: VariantHandle }[keyof D["variants"]]> + : C extends LocalContract + ? { readonly [K in keyof LocalMembers]: LocalHandleMember[K]> } + : never; + +export type Handle = HandleBase & MembersOf; diff --git a/packages/chord/src/declare.ts b/packages/chord/src/declare.ts new file mode 100644 index 000000000..c37e8d6a2 --- /dev/null +++ b/packages/chord/src/declare.ts @@ -0,0 +1,99 @@ +import type { Static, TSchema } from "typebox"; +import type { Cell, Sources, SourceValues } from "./cell.ts"; +import type { Ctx, Readable } from "./types.ts"; + +// Declaration descriptors. A factory passes them in object literals; the property name identifies the declaration +// across factory reruns (reload), independent of call order. + +const DECL = Symbol("chord.decl"); + +export interface Subscribable { + subscribe(listener: (event: E) => void): () => void; +} + +export interface CellDecl { + readonly [DECL]: "cell"; + readonly schema: TSchema; + readonly initial: T; +} + +export interface ResourceSpec { + /** Bump to replace the resource at the next reload. */ + readonly rev: number; + /** Holds or resources that must be live/open first. A change replaces the resource. */ + readonly deps?: readonly unknown[]; + /** Exclusive resources are replaced at the gate: old closed before new opened. */ + readonly exclusive?: boolean; + open(options: { readonly signal: AbortSignal }): T | Promise; + close?(value: T): void | Promise; +} + +export interface ResourceDecl { + readonly [DECL]: "resource"; + readonly spec: ResourceSpec; +} + +export interface Resource { + readonly key: string; + /** Throws not_ready until opened. */ + readonly value: T; +} + +export interface ListenerDecl { + readonly [DECL]: "listener"; + readonly source: unknown; + readonly select: ((value: never) => Subscribable) | undefined; + readonly fn: (event: unknown, ctx: Ctx) => void | Promise; +} + +export interface DerivationDecl { + readonly [DECL]: "derivation"; + readonly sources: Sources; + readonly fn: (...values: never[]) => T; +} + +/** A state cell: schema (required; there is no TypeScript at runtime) and initial value (validated). */ +export const cell = (schema: S, initial: Static): CellDecl> => ({ + [DECL]: "cell", + schema, + initial, +}); + +export const resource = (spec: ResourceSpec): ResourceDecl => ({ [DECL]: "resource", spec }); + +type ValueOf = S extends Resource ? T : S; +type EventOf = S extends Resource ? EventOf : S extends Subscribable ? E : never; + +/** + * Reacts to a source once the record is live. `listener(source, fn)` subscribes to the source itself: a readable + * (cell, derivation, handle state member: called with each value, starting with the current one), an event source, + * or a resource whose value is one. `listener(source, select, fn)` subscribes to `select(value)`, where value is the + * opened resource for a resource source. Each event runs `fn` in a fresh root. + */ +export function listener(source: S, fn: (event: EventOf, ctx: Ctx) => void | Promise): ListenerDecl; +export function listener( + source: S, + select: (value: ValueOf) => Subscribable, + fn: (event: E, ctx: Ctx) => void | Promise, +): ListenerDecl; +export function listener(source: unknown, ...rest: readonly unknown[]): ListenerDecl { + const [select, fn] = rest.length === 1 ? [undefined, rest[0]] : [rest[0], rest[1]]; + return { + [DECL]: "listener", + source, + select: select as ListenerDecl["select"], + fn: fn as ListenerDecl["fn"], + }; +} + +export const derivation = ( + sources: S, + fn: (...values: SourceValues) => T, +): DerivationDecl => ({ [DECL]: "derivation", sources, fn: fn as unknown as (...values: never[]) => T }); + +export const declKind = (value: unknown): string | undefined => + value !== null && typeof value === "object" ? (value as { [DECL]?: string })[DECL] : undefined; + +export type Cells = { readonly [K in keyof D]: D[K] extends CellDecl ? Cell : never }; +export type Resources = { readonly [K in keyof D]: D[K] extends ResourceDecl ? Resource : never }; +export type Derivations = { readonly [K in keyof D]: D[K] extends DerivationDecl ? Readable : never }; diff --git a/packages/chord/src/delta/README.md b/packages/chord/src/delta/README.md index db959c86b..171572623 100644 --- a/packages/chord/src/delta/README.md +++ b/packages/chord/src/delta/README.md @@ -35,7 +35,7 @@ const replica = applyImmutable(prepared.base, prepared.ops); | `tracker.value`, `prepared.base`, retained older revisions | No. | | `prepared.value`, `prepared.ops`, op tuples, paths, permutations, payloads | No. Payloads may be the same objects as parts of `prepared.value`. | | `applyImmutable()` / `applyImmutableBatches()` inputs and result | No. The result shares containers with both inputs. | -| Values from replicated state (`value`, listener values, loopback consumers) | No. In-process consumers may share the provider's containers. | +| Values from replicated state and cells (`value`, listener values, in-process handles) | No. In-process consumers may share the provider's containers. | | Mutable replica passed to `apply()` | Only through `apply()`, with batches it owns exclusively. Code that edits it between batches breaks convergence. | | Batch passed to `apply()` | Consumed. `apply()` adopts payload containers into the replica. Use a detached copy for exactly one replica and never touch it again. | @@ -183,7 +183,7 @@ apply error, discard the decoder and replica and recover from a later `r`. ## Limits and footguns - No freezing. Mutating any immutable value corrupts state silently. -- Loopback sharing: in-process service consumers may receive the provider's containers. A consumer mutation corrupts authority and makes remote replicas diverge. +- In-process sharing: local consumers may receive the provider's containers. A consumer mutation corrupts authority and makes remote replicas diverge. - A change inside a large flat array copies that array's pointer storage for the new revision. - Placement validation costs time and memory on bulk inserts. In a benchmark unshifting 100k three-field objects, it added about 19 ms and 20 MiB of transient heap. - External proxies as placements, argument coercion callbacks that mutate the draft, non-primitive array indices, and non-numeric comparator results are out of contract. So are sort comparators that mutate, prepare, abort, or adopt. Behavior is unspecified. diff --git a/packages/chord/src/delta/apply-immutable-trusted.ts b/packages/chord/src/delta/apply-immutable-trusted.ts index 9049ab040..7944bfae1 100644 --- a/packages/chord/src/delta/apply-immutable-trusted.ts +++ b/packages/chord/src/delta/apply-immutable-trusted.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "../types.ts"; +import type { JsonValue } from "../json.ts"; import type { Op, Path, Seg } from "./index.ts"; type JsonContainer = JsonValue[] | Record; diff --git a/packages/chord/src/delta/diff.ts b/packages/chord/src/delta/diff.ts index b022d142a..774ad1b59 100644 --- a/packages/chord/src/delta/diff.ts +++ b/packages/chord/src/delta/diff.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "../types.ts"; +import type { JsonValue } from "../json.ts"; import { type NonEmptyPath, type Op, overlap, type Path, RESERVED_SEGMENTS, type Seg } from "./index.ts"; const DEFAULT_OVERLAP_SCAN = 65_536; diff --git a/packages/chord/src/delta/index.ts b/packages/chord/src/delta/index.ts index 01c41ad1c..388c88514 100644 --- a/packages/chord/src/delta/index.ts +++ b/packages/chord/src/delta/index.ts @@ -1,6 +1,6 @@ -import type { JsonValue } from "../types.ts"; +import type { JsonValue } from "../json.ts"; -export type { JsonValue } from "../types.ts"; +export type { JsonValue } from "../json.ts"; const isObj = (value: unknown): value is object => value !== null && typeof value === "object"; diff --git a/packages/chord/src/delta/revision-validator.ts b/packages/chord/src/delta/revision-validator.ts index d78373e7a..4936dc9fe 100644 --- a/packages/chord/src/delta/revision-validator.ts +++ b/packages/chord/src/delta/revision-validator.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "../types.ts"; +import type { JsonValue } from "../json.ts"; type JsonContainer = JsonValue[] | Record; diff --git a/packages/chord/src/delta/tracker.ts b/packages/chord/src/delta/tracker.ts index c850127e2..8e01705fa 100644 --- a/packages/chord/src/delta/tracker.ts +++ b/packages/chord/src/delta/tracker.ts @@ -1,5 +1,5 @@ +import type { JsonValue } from "../json.ts"; import { copyJson } from "../json.ts"; -import type { JsonValue } from "../types.ts"; import { applyImmutableTrusted } from "./apply-immutable-trusted.ts"; import type { Draft } from "./draft.ts"; import { applyImmutable, type NonEmptyPath, type Op, overlap, type Path, RESERVED_SEGMENTS } from "./index.ts"; diff --git a/packages/chord/src/errors.ts b/packages/chord/src/errors.ts new file mode 100644 index 000000000..276b6ebf4 --- /dev/null +++ b/packages/chord/src/errors.ts @@ -0,0 +1,74 @@ +/** Error codes callers see (guide table). A refused or aborted reload keeps the code of its cause. */ +export type ChordErrorCode = + | "aborted" + | "contract_mismatch" + | "forbidden" + | "gone" + | "invalid" + | "not_in_variant" + | "not_ready" + | "released" + | "superseded" + | "unavailable" + | "upgrade_conflict" + | "upgrade_required"; + +/** + * Finer cause inside a code, for logs and tests: e.g. `invalid` + `args`, `result`, `state`, `input`, `behavior`, + * `duplicate`, `cycle`, `read_only`, `not_eager`, `registry_conflict`; `unavailable` + `unknown_type`. + */ +export type ChordErrorReason = string; + +const RETRYABLE: ReadonlySet = new Set(["superseded", "upgrade_conflict"]); + +export class ChordError extends Error { + readonly code: ChordErrorCode; + readonly reason: ChordErrorReason | undefined; + readonly retryable: boolean; + readonly details: unknown; + constructor( + code: ChordErrorCode, + message: string, + options?: { cause?: unknown; details?: unknown; reason?: ChordErrorReason }, + ) { + super(`${code}${options?.reason ? `/${options.reason}` : ""}: ${message}`, { + cause: options?.cause, + }); + this.name = "ChordError"; + this.code = code; + this.reason = options?.reason; + this.retryable = RETRYABLE.has(code); + this.details = options?.details; + } +} + +export const isChordError = (error: unknown, code?: ChordErrorCode): error is ChordError => + error instanceof ChordError && (code === undefined || error.code === code); + +/** For factories and methods refusing a principal: `throw forbidden()`. */ +export const forbidden = (message = "forbidden"): ChordError => new ChordError("forbidden", message); + +export const invalid = (reason: ChordErrorReason, message: string, details?: unknown): ChordError => + new ChordError("invalid", message, { reason, details }); + +/** Keeps a ChordError's code; anything else becomes `unavailable` with the original as cause. */ +export const asChordError = (error: unknown, context: string): ChordError => + error instanceof ChordError + ? error + : new ChordError("unavailable", `${context}: ${error instanceof Error ? error.message : String(error)}`, { + cause: error, + }); + +/** Runs every step, then throws one error, or an AggregateError when several failed. */ +export async function runAll(steps: readonly (() => unknown)[], message: string): Promise { + const errors: unknown[] = []; + for (const step of steps) { + try { + await step(); + } catch (error) { + errors.push(error); + } + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, message); +} diff --git a/packages/chord/src/facets/host.ts b/packages/chord/src/facets/host.ts deleted file mode 100644 index 7b8e5c0b1..000000000 --- a/packages/chord/src/facets/host.ts +++ /dev/null @@ -1,906 +0,0 @@ -import { BACKGROUND_CONTEXT } from "../context/index.ts"; -import { RemoteServiceBindingImpl } from "../services/consumer.ts"; -import { ServiceSlot } from "../services/handle.ts"; -import { InstanceDirectory, type InstanceDirectoryEntry } from "../services/instances.ts"; -import { createLoopbackServiceTransport } from "../services/loopback.ts"; -import { RemoteServiceProvider, validateRemoteServiceImplementation } from "../services/provider.ts"; -import { MutableReplicatedStateImpl } from "../services/state.ts"; -import type { - Context, - Facet, - FacetEnvironment, - FacetOptions, - RemoteServiceContract, - RemoteServiceSource, - RemoteServices, - Service, - ServiceMode, - ServiceSpawner, -} from "../types.ts"; - -interface FacetServiceReference { - readonly serviceId: string; - readonly service: Service; - readonly mode: ServiceMode; -} - -interface ExternalService { - readonly service: Service; - readonly mode: ServiceMode; - readonly source: RemoteServiceSource; -} - -interface FacetShape { - facetId: string; - requires: readonly FacetServiceReference[]; - provides: readonly FacetServiceReference[]; -} - -interface FacetRuntime extends FacetShape { - requires: FacetServiceReference[]; - provides: FacetServiceReference[]; - readonly lifecycle: FacetLifecycle; - readonly provisions: FacetProvision[]; - readonly singletonViews: Map; -} - -type LifecycleState = "setting_up" | "prepared" | "active" | "disposing" | "dead"; -type GenerationPhase = - | "setup" - | "assembling" - | "connecting" - | "activating" - | "active" - | "reloading" - | "disposing" - | "dead"; -type Disposal = () => void | Promise; - -class FacetLifecycle { - readonly id: string; - readonly #effects: Disposal[] = []; - readonly #observations: Array<() => Disposal> = []; - readonly #activate: Array<() => void | Promise> = []; - #state: LifecycleState = "setting_up"; - #serviceAccess = false; - - constructor(id: string) { - this.id = id; - } - - assertSettingUp(operation: string): void { - if (this.#state !== "setting_up") { - throw new Error(`Facet ${this.id} can ${operation} only during setup`); - } - } - - assertRunning(operation: string): void { - if (this.#state !== "setting_up" && this.#state !== "active") { - throw new Error(`Facet ${this.id} cannot ${operation} while ${this.#state}`); - } - } - - assertActive(operation: string): void { - if (this.#state !== "active") throw new Error(`Facet ${this.id} can ${operation} only while active`); - } - - assertServiceAccess(): void { - if (!this.#serviceAccess) { - throw new Error(`Facet ${this.id} service handles cannot be used while ${this.#state}`); - } - } - - revoke(): void { - this.#serviceAccess = false; - } - - own(disposal: Disposal): void { - this.assertRunning("own resources"); - this.#effects.push(disposal); - } - - observe(start: () => Disposal): void { - this.assertSettingUp("observe services"); - this.#observations.push(start); - } - - onActivate(callback: () => void | Promise): void { - this.assertSettingUp("register activation callbacks"); - this.#activate.push(callback); - } - - prepared(): void { - this.assertSettingUp("finish setup"); - this.#state = "prepared"; - } - - async activate(): Promise { - if (this.#state !== "prepared") throw new Error(`Facet ${this.id} is not prepared`); - this.#state = "active"; - this.#serviceAccess = true; - for (const start of this.#observations) this.#effects.push(start()); - for (const callback of this.#activate) await callback(); - } - - async dispose(): Promise { - if (this.#state === "dead") return; - this.#state = "disposing"; - const errors: unknown[] = []; - for (const effect of this.#effects.splice(0).reverse()) { - try { - await effect(); - } catch (error) { - errors.push(error); - } - } - this.#observations.length = 0; - this.#activate.length = 0; - this.#serviceAccess = false; - this.#state = "dead"; - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, `Failed to dispose facet ${this.id}`); - } -} - -interface KeyedServiceSource { - observe(service: Service, handler: (service: T, context: Context) => void | Promise): () => void; -} - -interface LocalKeyedRegistration { - readonly generations: Map; - readonly directory: InstanceDirectory; -} - -class LocalKeyedServiceRegistry implements KeyedServiceSource { - readonly #registrations = new Map(); - #disposed = false; - - constructor(services: readonly { readonly id: string }[], onError: (error: Error) => void) { - const ids = services.map(({ id }) => id); - if (new Set(ids).size !== ids.length) throw new TypeError("Local keyed service registry has duplicate IDs"); - for (const serviceId of ids) { - this.#registrations.set(serviceId, { - generations: new Map(), - directory: new InstanceDirectory({ ready: true, onError }), - }); - } - } - - spawn(service: Service, key: string, implementation: T): () => void { - this.#assertActive(); - if (key.length === 0) throw new TypeError("Local service instance key must not be empty"); - if (typeof implementation !== "object" || implementation === null || Array.isArray(implementation)) { - throw new TypeError(`Local service ${service.id} implementation must be an object`); - } - const registration = this.#registration(service.id); - if (registration.directory.get(key) !== undefined) { - throw new Error(`Local service ${service.id} already has a live instance with key ${key}`); - } - const generation = (registration.generations.get(key) ?? 0) + 1; - registration.generations.set(key, generation); - const instance: InstanceDirectoryEntry = { - key, - generation, - service: implementation as object, - deactivate() {}, - }; - registration.directory.insert(instance); - let closed = false; - return () => { - if (closed) return; - closed = true; - registration.directory.remove(instance); - }; - } - - observe(service: Service, handler: (service: T, context: Context) => void | Promise): () => void { - this.#assertActive(); - return this.#registration(service.id).directory.observe(handler); - } - - dispose(): void { - if (this.#disposed) return; - this.#disposed = true; - for (const registration of this.#registrations.values()) registration.directory.dispose(); - this.#registrations.clear(); - } - - #registration(serviceId: string): LocalKeyedRegistration { - const registration = this.#registrations.get(serviceId); - if (registration === undefined) throw new Error(`Local keyed service ${serviceId} is not registered`); - return registration; - } - - #assertActive(): void { - if (this.#disposed) throw new Error("Local keyed service registry is disposed"); - } -} - -class HostServiceSlots { - readonly #singletons = new Map(); - readonly #keyedSources = new Map(); - - getSingleton(service: Service, assertAccess: () => void): T { - let slot = this.#singletons.get(service.id); - if (slot === undefined) { - slot = new ServiceSlot(service.id, !service.local); - this.#singletons.set(service.id, slot); - } - return slot.view(assertAccess); - } - - hasSingleton(serviceId: string): boolean { - return this.#singletons.has(serviceId); - } - - observe( - service: Service, - assertAccess: () => void, - handler: (service: T, context: Context) => void | Promise, - ): Disposal { - const source = this.#keyedSources.get(service.id); - if (source === undefined) throw new Error(`Service ${service.id} is disconnected`); - let stopped = false; - const stop = source.observe(service, (target, context) => { - const slot = new ServiceSlot(service.id, !service.local); - slot.bind(target as object); - return handler( - slot.view(() => { - assertAccess(); - if (stopped || context.abortSignal?.aborted) { - throw new Error(`Keyed service ${service.id} observation is closed`); - } - }), - context, - ); - }); - return () => { - if (stopped) return; - stopped = true; - stop(); - }; - } - - bindSingleton(serviceId: string, target: object): void { - this.#singletons.get(serviceId)?.bind(target); - } - - bindKeyed(serviceId: string, services: KeyedServiceSource): void { - this.#keyedSources.set(serviceId, services); - } - - dispose(): void { - for (const slot of this.#singletons.values()) slot.unbind(); - this.#singletons.clear(); - this.#keyedSources.clear(); - } -} - -type ServiceInstanceInstaller = (key: string, implementation: T) => () => void; - -interface StagedServiceInstance { - readonly key: string; - readonly implementation: T; - release?: () => void; -} - -class StagedServiceSpawner implements ServiceSpawner { - readonly #lifecycle: FacetLifecycle; - readonly #validate: (key: string, implementation: T) => void; - readonly #instances = new Map>(); - #installer: ServiceInstanceInstaller | undefined; - - constructor(lifecycle: FacetLifecycle, validate: (key: string, implementation: T) => void) { - this.#lifecycle = lifecycle; - this.#validate = validate; - } - - connect(installer: ServiceInstanceInstaller): void { - if (this.#installer !== undefined) throw new Error("Facet service provider is already connected"); - this.#installer = installer; - for (const instance of this.#instances.values()) { - instance.release = installer(instance.key, instance.implementation); - } - } - - spawn(key: string, implementation: T): () => void { - this.#lifecycle.assertActive("spawn service instances"); - this.#validate(key, implementation); - if (this.#instances.has(key)) throw new Error(`Facet service already has a live instance with key ${key}`); - const instance: StagedServiceInstance = { key, implementation }; - this.#instances.set(key, instance); - if (this.#installer !== undefined) instance.release = this.#installer(key, implementation); - const close = (): void => { - if (this.#instances.get(key) !== instance) return; - this.#instances.delete(key); - instance.release?.(); - }; - this.#lifecycle.own(close); - return close; - } -} - -type FacetProvision = - | { - readonly kind: "singleton"; - readonly service: { readonly id: string; readonly local: boolean }; - readonly implementation: object; - install(provider: RemoteServiceProvider): void; - validateReplacement(provider: RemoteServiceProvider): void; - replace(provider: RemoteServiceProvider): void; - } - | { - readonly kind: "keyed"; - readonly service: { readonly id: string; readonly local: boolean }; - connectLocal(registry: LocalKeyedServiceRegistry): void; - connectRemote(provider: RemoteServiceProvider): void; - }; - -/** Private lifecycle and dependency kernel behind the atomic host entry point. */ -export class FacetKernel { - readonly #initialFacets: readonly Facet[]; - readonly #serviceSources: readonly RemoteServiceSource[]; - readonly #onError: (error: Error) => void; - readonly #facets = new Map(); - readonly #serviceSlots: HostServiceSlots; - readonly #sourceBindings = new Map(); - #activationOrder: readonly string[] = []; - #provider: RemoteServiceProvider | undefined; - #internalServices: RemoteServices | undefined; - #localKeyedServices: LocalKeyedServiceRegistry | undefined; - #phase: GenerationPhase = "setup"; - - constructor(options: FacetOptions) { - const ids = options.facets.map((facet) => facet.id); - if (ids.some((id) => id.length === 0)) throw new Error("Facet ID must not be empty"); - if (new Set(ids).size !== ids.length) throw new Error("Facet IDs must be unique within a generation"); - this.#initialFacets = options.facets; - this.#serviceSources = options.serviceSources ?? []; - this.#onError = options.onError ?? (() => {}); - this.#serviceSlots = new HostServiceSlots(); - } - - get provider(): RemoteServiceProvider { - if (this.#provider === undefined) throw new Error("Facet service provider is not assembled"); - return this.#provider; - } - - #createFacetRuntime(facetId: string): FacetRuntime { - return { - facetId, - requires: [], - provides: [], - lifecycle: new FacetLifecycle(facetId), - provisions: [], - singletonViews: new Map(), - }; - } - - #setupFacet(facet: Facet, record: FacetRuntime): void { - const result: unknown = facet.setup(this.#environment(record)); - if (isPromiseLike(result)) { - void Promise.resolve(result).catch(() => {}); - throw new Error(`Facet ${facet.id} setup must be synchronous`); - } - record.lifecycle.prepared(); - } - - async activate(): Promise { - const records: FacetRuntime[] = []; - try { - for (const facet of this.#initialFacets) { - const record = this.#createFacetRuntime(facet.id); - this.#facets.set(facet.id, record); - this.#setupFacet(facet, record); - records.push(record); - } - - this.#phase = "assembling"; - const externalServices = await this.#resolveExternalServices(records); - this.#activationOrder = validateFacets(records, externalServices); - this.#assembleProviders(); - this.#bindServices(externalServices); - - this.#phase = "connecting"; - await Promise.all( - [...this.#sourceBindings.values(), this.#internalServiceBinding].map((services) => - services.ready(BACKGROUND_CONTEXT), - ), - ); - - this.#phase = "activating"; - for (const id of this.#activationOrder) await this.#facets.get(id)!.lifecycle.activate(); - this.#phase = "active"; - } catch (error) { - const cleanupErrors = await this.#terminate(); - if (cleanupErrors.length > 0) { - throw new AggregateError([error, ...cleanupErrors], "Facet generation startup and cleanup failed"); - } - throw error; - } - } - - async reload(facets: readonly Facet[]): Promise { - if (this.#phase !== "active") throw new Error(`Facet host cannot reload while ${this.#phase}`); - const ids = facets.map(({ id }) => id); - if (ids.some((id) => id.length === 0)) throw new Error("Facet ID must not be empty"); - if (new Set(ids).size !== ids.length) throw new Error("Reloaded facet IDs must be unique"); - for (const id of ids) { - if (!this.#facets.has(id)) throw new Error(`Facet ${id} is not active`); - } - this.#phase = "reloading"; - - const staged: FacetRuntime[] = []; - const candidates: FacetRuntime[] = []; - try { - for (const facet of facets) { - const record = this.#createFacetRuntime(facet.id); - staged.push(record); - this.#setupFacet(facet, record); - const previous = this.#facets.get(facet.id)!; - if (!sameFacetShape(previous, record)) { - throw new Error(`Reloaded facet ${facet.id} must preserve its service requirements and provisions`); - } - this.#validateReplacementProvisions(record.provisions); - candidates.push(record); - } - } catch (error) { - const cleanupErrors = await disposeFacetRecords(staged.reverse()); - if (cleanupErrors.length > 0) { - const abortErrors = await this.#abort(); - throw new AggregateError( - [error, ...cleanupErrors, ...abortErrors], - "Facet reload setup and cleanup failed", - ); - } - this.#phase = "active"; - throw error; - } - - const replacements = new Map(candidates.map((record) => [record.facetId, record])); - const candidateOrder = this.#activationOrder.flatMap((id) => { - const candidate = replacements.get(id); - return candidate === undefined ? [] : [candidate]; - }); - try { - for (const candidate of candidateOrder) await candidate.lifecycle.activate(); - for (const candidate of candidateOrder) this.#validateReplacementProvisions(candidate.provisions); - } catch (error) { - const cleanupErrors = await disposeFacetRecords([...candidateOrder].reverse()); - if (cleanupErrors.length > 0) { - const abortErrors = await this.#abort(); - throw new AggregateError( - [error, ...cleanupErrors, ...abortErrors], - "Facet reload activation and cleanup failed", - ); - } - this.#phase = "active"; - throw error; - } - - const previous = candidateOrder.map(({ facetId }) => this.#facets.get(facetId)!); - for (const candidate of candidateOrder) this.#facets.set(candidate.facetId, candidate); - try { - for (const candidate of candidateOrder) { - for (const provision of candidate.provisions) { - if (provision.kind !== "singleton") continue; - if (provision.service.local) { - this.#serviceSlots.bindSingleton(provision.service.id, provision.implementation); - } else { - provision.replace(this.provider); - } - } - } - const retirementErrors = await disposeFacetRecords(previous.reverse()); - if (retirementErrors.length === 1) throw retirementErrors[0]; - if (retirementErrors.length > 1) { - throw new AggregateError(retirementErrors, "Failed to retire replaced facets"); - } - for (const candidate of candidateOrder) { - for (const provision of candidate.provisions) { - if (provision.kind !== "keyed") continue; - if (provision.service.local) provision.connectLocal(this.#localKeyedRegistry); - else provision.connectRemote(this.provider); - } - } - } catch (error) { - const abortErrors = await this.#abort(previous); - throw new AggregateError([error, ...abortErrors], "Facet reload failed after cutover"); - } - this.#phase = "active"; - } - - async dispose(): Promise { - if (this.#phase === "dead") return; - if (this.#phase !== "active") throw new Error(`Facet host cannot be disposed while ${this.#phase}`); - const errors = await this.#terminate(); - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, "Failed to dispose facet generation"); - } - - #validateReplacementProvisions(provisions: readonly FacetProvision[]): void { - for (const provision of provisions) { - if (provision.kind !== "singleton" || provision.service.local) continue; - provision.validateReplacement(this.provider); - } - } - - #environment(runtime: FacetRuntime): FacetEnvironment { - const { lifecycle, provisions } = runtime; - return { - provide: (service: Service, implementation: NoInfer): void => { - lifecycle.assertSettingUp("provide services"); - if (typeof implementation !== "object" || implementation === null || Array.isArray(implementation)) { - throw new TypeError(`Service ${service.id} implementation must be an object`); - } - recordServiceReference(runtime.provides, service, "singleton"); - provisions.push({ - kind: "singleton", - service, - implementation, - install: (provider) => provider.provide(service, implementation as NoInfer>), - validateReplacement: (provider) => - provider.validateReplacement(service, implementation as NoInfer>), - replace: (provider) => provider.replace(service, implementation as NoInfer>), - }); - }, - provideMany: (service: Service): ServiceSpawner => { - lifecycle.assertSettingUp("provide service instances"); - recordServiceReference(runtime.provides, service, "keyed"); - const instances = new StagedServiceSpawner(lifecycle, (key, implementation) => { - if (key.length === 0) throw new TypeError("Facet service instance key must not be empty"); - if (typeof implementation !== "object" || implementation === null || Array.isArray(implementation)) { - throw new TypeError(`Facet service ${service.id} implementation must be an object`); - } - if (!service.local) validateRemoteServiceImplementation(service.id, implementation); - }); - provisions.push({ - kind: "keyed", - service, - connectLocal: (registry) => - instances.connect((key, implementation) => registry.spawn(service, key, implementation)), - connectRemote: (provider) => - instances.connect((key, implementation) => - provider.spawn(service, key, implementation as NoInfer>), - ), - }); - return instances; - }, - use: (service: Service): T => { - lifecycle.assertSettingUp("acquire services"); - recordServiceReference(runtime.requires, service, "singleton"); - let view = runtime.singletonViews.get(service.id); - if (view === undefined) { - view = this.#serviceSlots.getSingleton(service, () => lifecycle.assertServiceAccess()); - runtime.singletonViews.set(service.id, view); - } - return view as T; - }, - observe: (service: Service, handler: (service: T, context: Context) => void | Promise): void => { - lifecycle.assertSettingUp("observe services"); - recordServiceReference(runtime.requires, service, "keyed"); - lifecycle.observe(() => - this.#serviceSlots.observe(service, () => lifecycle.assertServiceAccess(), handler), - ); - }, - replicatedState: (initial: T) => { - lifecycle.assertRunning("create replicated state"); - return new MutableReplicatedStateImpl(initial); - }, - own: (disposal) => lifecycle.own(disposal), - onActivate: (callback) => lifecycle.onActivate(callback), - onDeactivate: (callback) => lifecycle.own(callback), - }; - } - - async #resolveExternalServices(records: readonly FacetShape[]): Promise> { - const catalogues = await Promise.all( - this.#serviceSources.map(async (source) => ({ - source, - entries: await source.catalogue(BACKGROUND_CONTEXT), - })), - ); - const offered = new Map(); - for (const { source, entries } of catalogues) { - for (const { serviceId, mode } of entries) { - if (offered.has(serviceId)) { - throw new Error(`Facet host service ${serviceId} is offered by more than one source`); - } - offered.set(serviceId, { mode, source }); - } - } - - const local = new Set(records.flatMap(({ provides }) => provides.map(({ serviceId }) => serviceId))); - const external = new Map(); - for (const { requires } of records) { - for (const requirement of requires) { - if (local.has(requirement.serviceId) || external.has(requirement.serviceId)) continue; - let source = offered.get(requirement.serviceId); - if (source === undefined) { - const deferred = this.#serviceSources.filter( - ({ acceptsUnavailableServices }) => acceptsUnavailableServices, - ); - if (deferred.length > 1) { - throw new Error(`Facet host service ${requirement.serviceId} has more than one deferred source`); - } - if (deferred.length === 1) source = { mode: requirement.mode, source: deferred[0]! }; - } - if (source !== undefined) { - external.set(requirement.serviceId, { ...source, service: requirement.service }); - } - } - } - const serviceIdsBySource = new Map(); - for (const [serviceId, { source }] of external) { - let serviceIds = serviceIdsBySource.get(source); - if (serviceIds === undefined) { - serviceIds = []; - serviceIdsBySource.set(source, serviceIds); - } - serviceIds.push(serviceId); - } - for (const [source, serviceIds] of serviceIdsBySource) { - this.#sourceBindings.set( - source, - source.open({ - services: serviceIds.map((id) => ({ id })), - assertAccess: () => this.#assertServiceTargetAccess(), - onError: this.#onError, - }), - ); - } - return external; - } - - #assembleProviders(): void { - const provisions = this.#provisions(); - const remoteProvisions = provisions.filter(({ service }) => !service.local); - const provider = new RemoteServiceProvider( - remoteProvisions.map(({ service, kind }) => ({ service, mode: kind })), - ); - const internalServices = new RemoteServiceBindingImpl({ - services: remoteProvisions.map(({ service }) => service), - transport: createLoopbackServiceTransport(provider), - assertAccess: () => this.#assertServiceTargetAccess(), - onError: this.#onError, - }); - const localKeyedServices = new LocalKeyedServiceRegistry( - provisions.flatMap((provision) => - provision.kind === "keyed" && provision.service.local ? [provision.service] : [], - ), - this.#onError, - ); - this.#provider = provider; - this.#internalServices = internalServices; - this.#localKeyedServices = localKeyedServices; - for (const provision of provisions) { - if (provision.kind === "singleton") { - if (!provision.service.local) provision.install(provider); - } else if (provision.service.local) { - provision.connectLocal(localKeyedServices); - } else { - provision.connectRemote(provider); - } - } - } - - #bindServices(externalServices: ReadonlyMap): void { - for (const provision of this.#provisions()) { - if (provision.kind === "singleton") { - if (!this.#serviceSlots.hasSingleton(provision.service.id)) continue; - const target = provision.service.local - ? provision.implementation - : (this.#internalServiceBinding.use(provision.service) as object); - this.#serviceSlots.bindSingleton(provision.service.id, target); - } else { - this.#serviceSlots.bindKeyed( - provision.service.id, - provision.service.local ? this.#localKeyedRegistry : this.#internalServiceBinding, - ); - } - } - for (const [serviceId, { service, mode, source }] of externalServices) { - const services = this.#sourceBindings.get(source); - if (services === undefined) throw new Error(`Service source for ${serviceId} is not open`); - if (mode === "singleton") { - this.#serviceSlots.bindSingleton(serviceId, services.use(service) as object); - } else { - this.#serviceSlots.bindKeyed(serviceId, services); - } - } - } - - #provisions(): FacetProvision[] { - return [...this.#facets.values()].flatMap(({ provisions }) => provisions); - } - - get #localKeyedRegistry(): LocalKeyedServiceRegistry { - if (this.#localKeyedServices === undefined) throw new Error("Facet keyed services are not assembled"); - return this.#localKeyedServices; - } - - get #internalServiceBinding(): RemoteServices { - if (this.#internalServices === undefined) throw new Error("Facet remote services are not assembled"); - return this.#internalServices; - } - - async #disposeServiceBindings(): Promise { - const bindings = [...this.#sourceBindings.values()]; - this.#sourceBindings.clear(); - if (this.#internalServices !== undefined) bindings.push(this.#internalServices); - this.#internalServices = undefined; - const results = await Promise.allSettled(bindings.map((services) => services.dispose(BACKGROUND_CONTEXT))); - return results.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])); - } - - #assertServiceTargetAccess(): void { - if ( - this.#phase !== "activating" && - this.#phase !== "active" && - this.#phase !== "reloading" && - this.#phase !== "disposing" - ) { - throw new Error(`Facet service targets cannot be used during ${this.#phase}`); - } - } - - async #abort(extraRecords: readonly FacetRuntime[] = []): Promise { - for (const record of this.#facets.values()) record.lifecycle.revoke(); - for (const record of extraRecords) record.lifecycle.revoke(); - return this.#terminate(extraRecords); - } - - async #terminate(extraRecords: readonly FacetRuntime[] = []): Promise { - this.#phase = "disposing"; - const errors = await this.#disposeLifecycles(); - errors.push(...(await disposeFacetRecords([...extraRecords].reverse()))); - try { - this.#localKeyedServices?.dispose(); - } catch (error) { - errors.push(error); - } - this.#localKeyedServices = undefined; - errors.push(...(await this.#disposeServiceBindings())); - try { - this.#serviceSlots.dispose(); - } catch (error) { - errors.push(error); - } - try { - this.#provider?.dispose(); - } catch (error) { - errors.push(error); - } - this.#phase = "dead"; - return errors; - } - - async #disposeLifecycles(): Promise { - const errors: unknown[] = []; - const order = - this.#activationOrder.length > 0 ? [...this.#activationOrder].reverse() : [...this.#facets.keys()].reverse(); - for (const id of order) { - const record = this.#facets.get(id); - if (record === undefined) continue; - this.#facets.delete(id); - try { - await record.lifecycle.dispose(); - } catch (error) { - errors.push(error); - } - } - return errors; - } -} - -async function disposeFacetRecords(records: readonly FacetRuntime[]): Promise { - const errors: unknown[] = []; - for (const record of records) { - try { - await record.lifecycle.dispose(); - } catch (error) { - errors.push(error); - } - } - return errors; -} - -function validateFacets( - records: readonly FacetShape[], - externalServices: ReadonlyMap, -): string[] { - const providers = new Map< - string, - { readonly facetId: string | undefined; readonly mode: ServiceMode | undefined } - >(); - for (const [serviceId, { mode }] of externalServices) providers.set(serviceId, { facetId: undefined, mode }); - for (const record of records) { - for (const provision of record.provides) { - const existing = providers.get(provision.serviceId); - if (existing !== undefined) { - if (existing.mode !== undefined && existing.mode !== provision.mode) { - throw new Error(`Service ${provision.serviceId} is provided as both singleton and keyed`); - } - if (existing.facetId === undefined) { - throw new Error(`Service ${provision.serviceId} is provided by both the host and ${record.facetId}`); - } - throw new Error( - `Service ${provision.serviceId} is provided by both ${existing.facetId} and ${record.facetId}`, - ); - } - providers.set(provision.serviceId, { facetId: record.facetId, mode: provision.mode }); - } - } - - const dependencies = new Map(records.map((record) => [record.facetId, new Set()])); - const dependents = new Map(records.map((record) => [record.facetId, new Set()])); - for (const record of records) { - for (const requirement of record.requires) { - const provider = providers.get(requirement.serviceId); - if (provider === undefined) { - throw new Error( - `Facet ${record.facetId} requires local/${requirement.serviceId}/${requirement.mode}, but no facet provides it`, - ); - } - if (provider.mode !== undefined && provider.mode !== requirement.mode) { - throw new Error( - `Facet ${record.facetId} requires ${requirement.serviceId} as ${requirement.mode}, but ${provider.facetId ?? "the host"} provides it as ${provider.mode}`, - ); - } - if (provider.facetId === undefined || provider.facetId === record.facetId) continue; - dependencies.get(record.facetId)!.add(provider.facetId); - dependents.get(provider.facetId)!.add(record.facetId); - } - } - return topologicalOrder(records, dependencies, dependents); -} - -function topologicalOrder( - records: readonly FacetShape[], - dependencies: ReadonlyMap>, - dependents: ReadonlyMap>, -): string[] { - const remaining = new Map([...dependencies].map(([id, values]) => [id, values.size])); - const ready = records.map((record) => record.facetId).filter((id) => remaining.get(id) === 0); - const order: string[] = []; - while (ready.length > 0) { - const id = ready.shift()!; - order.push(id); - for (const dependent of dependents.get(id) ?? []) { - const count = remaining.get(dependent)! - 1; - remaining.set(dependent, count); - if (count === 0) ready.push(dependent); - } - } - if (order.length !== records.length) { - const cycle = records.map((record) => record.facetId).filter((id) => remaining.get(id)! > 0); - throw new Error(`Facet dependency cycle: ${cycle.join(", ")}`); - } - return order; -} - -function recordServiceReference( - target: FacetServiceReference[], - service: { readonly id: string }, - mode: ServiceMode, -): void { - if (target.some((reference) => reference.serviceId === service.id && reference.mode === mode)) return; - target.push({ serviceId: service.id, service: service as Service, mode }); -} - -function sameFacetShape(left: FacetShape, right: FacetShape): boolean { - return sameReferences(left.requires, right.requires) && sameReferences(left.provides, right.provides); -} - -function sameReferences(left: readonly FacetServiceReference[], right: readonly FacetServiceReference[]): boolean { - return ( - left.length === right.length && - left.every((reference) => - right.some((other) => other.serviceId === reference.serviceId && other.mode === reference.mode), - ) - ); -} - -function isPromiseLike(value: unknown): value is PromiseLike { - return typeof value === "object" && value !== null && "then" in value && typeof value.then === "function"; -} diff --git a/packages/chord/src/facets/loader.ts b/packages/chord/src/facets/loader.ts deleted file mode 100644 index 0e4374abc..000000000 --- a/packages/chord/src/facets/loader.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { LoadedFacets } from "../types.ts"; - -export async function disposeLoadedFacets(loaded: readonly LoadedFacets[]): Promise { - const results = await Promise.allSettled(loaded.map((entry) => entry.dispose())); - return results.flatMap((result) => (result.status === "rejected" ? [result.reason] : [])); -} diff --git a/packages/chord/src/host.ts b/packages/chord/src/host.ts new file mode 100644 index 000000000..93d753298 --- /dev/null +++ b/packages/chord/src/host.ts @@ -0,0 +1,1010 @@ +import { ChildReloads, reconcileChild, runClusterReload } from "./cluster.ts"; +import { type AcceptOptions, Connection, type LinkOptions, type RelayHold } from "./connection.ts"; +import { + type AnyContract, + bind, + type Handle, + type InputOf, + type LocalContract, + type LocalDef, + type RemoteContract, + type RemoteDef, +} from "./contract.ts"; +import { ChordError, invalid, runAll } from "./errors.ts"; +import type { Link } from "./link.ts"; +import { enqueue } from "./publish.ts"; +import { + CtxImpl, + type Decls, + type Factory, + HandleImpl, + type Hold, + type HoldOwner, + type HostCore, + makeService, + RecordImpl, + type ServeOptions, + type Service, +} from "./record.ts"; +import type { Registry, RegistryTable, RegistryView } from "./registry.ts"; +import { type ReloadOptions, type ReloadReport, runReload } from "./reload.ts"; +import { RemoteHandle } from "./remote.ts"; +import { canonicalIdentity, canonicalJson, deepEqual, normalize } from "./schema.ts"; +import type { Ctx, GetOptions, Principal } from "./types.ts"; +import type { Frame } from "./wire.ts"; + +/** The principal of in-host code. */ +export const hostPrincipal: Principal = Object.freeze({ host: true }); + +/** + * Registers a factory for a type. Overloaded on the contract's definition (not the contract type) so TypeScript + * keeps literal `kind`s in factory returns. + */ +export interface Serve { + ( + type: RemoteContract, + factory: Factory>, + options?: ServeOptions>, + ): void; + ( + type: LocalContract, + factory: Factory>, + options?: ServeOptions>, + ): void; +} + +type AnyServe = (type: AnyContract, factory: Factory, options?: ServeOptions) => void; + +/** + * What a facet's `setup` gets. `serve` registers types; `root` and `registry` give the facet's code that is not a + * Chord method, listener or task (Durable tools and hooks, callbacks of native libraries) access to its own host, + * so plugins never need a process-global host. Both may be kept and used after `setup` returns. + */ +export interface FacetApi { + readonly serve: Serve; + /** The host's name (`server`, `session:S1`, ...). */ + readonly name: string; + /** Starts a root on the facet's host (in its current epoch at call time). */ + root(fn: (ctx: Ctx) => T | Promise, options?: RootOptions): Promise; + /** The facet's host's registry view (`get(key, ctx)` reads the table of `ctx.epoch`). */ + registry(registry: Registry): RegistryView; +} + +/** What a facet module exports (default export): the code that serves its types on one host kind. */ +export interface FacetDefinition { + /** Contract tokens this facet consumes; checked against providers at reload. */ + readonly uses?: readonly AnyContract[]; + setup(api: FacetApi): void; +} + +/** A named facet. Modules are named by their URL; reloading a facet replaces everything it served. */ +export interface Facet extends FacetDefinition { + readonly name: string; +} + +export const defineFacet = ( + setup: (api: FacetApi) => void, + options: { readonly uses?: readonly AnyContract[] } = {}, +): FacetDefinition => ({ setup, uses: options.uses }); + +/** Imports facet modules (fresh copies on every call) and names them by module URL. */ +export type FacetLoader = (modules: readonly (string | URL)[]) => Promise; + +/** A facet object, or a module (path or URL) whose default export is a FacetDefinition. */ +export type PluginRef = Facet | string | URL; + +const isFacet = (ref: PluginRef): ref is Facet => typeof ref === "object" && !(ref instanceof URL); + +export interface HostOptions { + readonly name?: string; + /** Maps a principal to its record key (should include connection provenance and grant revision). */ + readonly principalKey?: (principal: Principal) => string; + /** Validates every cell write and exposed state value against its schemas. */ + readonly dev?: boolean; + /** Errors from background work (disposal, listeners, tasks). Defaults to console.error. */ + readonly onError?: (error: unknown) => void; + /** Needed for `load`/`reload` with modules; `nodeLoader` from loader.ts in Node. */ + readonly loader?: FacetLoader; + /** `client` (browser): never a reload participant, never carries roots or epochs. Default `host`. */ + readonly kind?: "host" | "client"; + /** The link to this host's parent (server). Can also be set later with `connect`. */ + readonly upstream?: Link; + readonly upstreamOptions?: LinkOptions; +} + +/** Where requests for hosts named `prefix…` go (on the server). */ +export interface Route { + /** Checked before resolving (or spawning) and again after: grants may change meanwhile. */ + authorize(principal: Principal, hostName: string): boolean | Promise; + /** A link to the host, spawned if needed. */ + resolve(hostName: string): Promise; + /** Principal the resolved host acts as when it calls back (default `{ host: hostName }`). */ + readonly principal?: (hostName: string) => Principal; +} + +/** `invalidatePrincipal` matches: a predicate (this host only), or a pattern (fields that must be equal; propagates). */ +export type PrincipalMatch = ((principal: Principal) => boolean) | Readonly>; + +export interface RootOptions { + readonly principal?: Principal; + readonly signal?: AbortSignal; + /** Start the root in a retained epoch (links carry epochs). Defaults to the current epoch. */ + readonly epoch?: number; +} + +export interface LoadOptions { + /** Await eager activation (default true). False starts it but leaves readiness to `host.ready()`. */ + readonly awaitEager?: boolean; +} + +export interface Host { + readonly name: string; + readonly epoch: number; + readonly previousEpoch: number | undefined; + readonly serve: Serve; + /** Installs synchronously, then awaits this facet's eager objects unless `awaitEager` is false. */ + install(facet: Facet, options?: LoadOptions): Promise; + /** Installs all facets before awaiting their eager objects; lazy objects stay lazy. */ + load(plugins: readonly PluginRef[], options?: LoadOptions): Promise; + /** Awaits the eager holds present at call time, including those started by `serve` or deferred loads. */ + ready(): Promise; + get(type: C, input: InputOf, options?: GetOptions): Handle; + root(fn: (ctx: Ctx) => T | Promise, options?: RootOptions): Promise; + registry(registry: Registry): RegistryView; + /** + * Force-disposes every record whose principal matches (never the host principal), re-stamps or closes matching + * links, and propagates pattern matches to child hosts. + */ + invalidatePrincipal(match: PrincipalMatch): Promise; + /** Accepts a child link (a worker or a browser); requests on it act as `options.principal`. */ + accept(link: Link, options: AcceptOptions): void; + /** Sets (or replaces, after a reconnect) the link to this host's parent; remote handles re-hold over it. */ + connect(link: Link, options?: LinkOptions): void; + route(prefix: string, route: Route): void; + /** Reloads the given facets (modules are re-imported); facets not listed keep running unchanged. */ + reload(plugins: readonly PluginRef[], options?: ReloadOptions): Promise; + close(): Promise; +} + +export interface EpochState { + readonly id: number; + readonly services: Map; + tables: ReadonlyMap>; + /** Running roots with the number of their calls running here. */ + readonly roots: Map; +} + +const EMPTY_TABLE: RegistryTable = new Map(); + +export function createHost(options: HostOptions = {}): Host { + return new HostImpl(options); +} + +export class HostImpl implements Host, HostCore { + readonly name: string; + readonly dev: boolean; + readonly principal: Principal = hostPrincipal; + inFactory = 0; + failed: unknown; + readonly records = new Map(); + readonly epochs = new Map(); + readonly facets = new Map(); + readonly eager = new Map(); + readonly kind: "host" | "client"; + readonly connections = new Set(); + upstream: Connection | undefined; + /** Remote handles this host holds (re-held after an upstream reconnect). */ + readonly remotes = new Set(); + readonly #routes = new Map(); + readonly #childReloads = new ChildReloads(this); + /** Child facet modules by host-name prefix, from every committed cluster reload (for reconciling children). */ + remoteModules: Record = {}; + /** Removal tombstones also reconcile children that connected during a transaction. */ + remoteRemoved: Record = {}; + /** Highest epoch the parent has retired. */ + #retireAllowed = 0; + /** Roots that came over a link, by the link they came from. */ + /** Foreign roots (started on another host), by the links they came over. */ + readonly #foreign = new Map>(); + /** Roots started on this host that have not ended. */ + readonly #local = new Set(); + readonly #incarnation = Math.random().toString(36).slice(2, 8); + /** Links each root's calls went out on. */ + readonly #rootLinks = new Map>(); + current: EpochState; + previous: EpochState | undefined; + /** The epoch a running reload will commit; new types activated during prepare install their behavior for it. */ + pending: EpochState | undefined; + /** Activations of these types wait (reload gate). */ + /** Types behind a reload gate: new records of these types are rejected `upgrade_conflict` until commit/abort. */ + gating: ReadonlySet | undefined; + reloading = false; + retired: () => void = () => {}; + readonly #principalKey: (principal: Principal) => string; + readonly #onError: (error: unknown) => void; + readonly #loader: FacetLoader | undefined; + readonly #dying = new Map(); + readonly #touches = new Map>(); + readonly #order = new Map>(); + readonly #subscribers = new Map, epoch: number) => void>>(); + #epochSeq = 1; + #rootSeq = 0; + + constructor(options: HostOptions) { + this.name = options.name ?? "host"; + this.dev = options.dev ?? false; + this.#principalKey = options.principalKey ?? canonicalJson; + this.#onError = options.onError ?? ((error) => console.error("chord:", error)); + this.#loader = options.loader; + this.current = { id: this.#epochSeq, services: new Map(), tables: new Map(), roots: new Map() }; + this.epochs.set(this.current.id, this.current); + this.kind = options.kind ?? "host"; + if (options.upstream !== undefined) this.connect(options.upstream, options.upstreamOptions); + } + + get epoch(): number { + return this.current.id; + } + + get previousEpoch(): number | undefined { + return this.previous?.id; + } + + /** Allocates the next epoch id, or adopts one issued by the coordinating server. */ + nextEpochId(given?: number): number { + if (given !== undefined && given <= this.current.id) + throw new Error(`epoch ${given} is not newer than ${this.current.id}`); + this.#epochSeq = given ?? this.#epochSeq + 1; + return this.#epochSeq; + } + + report(error: unknown): void { + this.#onError(error); + } + + // ─── serving ─── + + readonly serve: Serve = ((type, factory, options = {}) => { + if (this.reloading) throw new Error("host.serve during a reload; serve from a facet passed to reload()"); + this.serveInto(this.current.services, makeService(type, factory, options, "")); + if (options.eager) this.startEager(this.current.services.get(type.name)!); + }) satisfies AnyServe as Serve; + + install(facet: Facet, options: LoadOptions = {}): Promise { + if (this.reloading) throw new Error("host.install during a reload"); + if (this.facets.has(facet.name)) throw new Error(`facet ${facet.name} is already installed; use reload()`); + const added: Service[] = []; + facet.setup( + this.facetApi(((type, factory, options = {}) => { + const service = makeService(type, factory, options, facet.name); + this.serveInto(this.current.services, service); + added.push(service); + }) satisfies AnyServe as Serve), + ); + this.facets.set(facet.name, facet); + const holds = added.filter((service) => service.options.eager).map((service) => this.startEager(service)); + return options.awaitEager === false ? Promise.resolve() : this.#eagerReady(holds); + } + + async load(plugins: readonly PluginRef[], options: LoadOptions = {}): Promise { + const facets = await this.resolve(plugins); + // No await between installs: an eager factory may consume a type served by a later facet. + await Promise.all(facets.map((facet) => this.install(facet, options))); + } + + ready(): Promise { + return this.#eagerReady(this.eager.values()); + } + + #eagerReady(holds: Iterable): Promise { + const ready = Promise.all([...holds].map((hold) => hold.ready)).then(() => {}); + // Ignoring install's return still reports through onError without an unhandled rejection. + ready.catch(() => {}); + return ready; + } + + /** Turns plugin references into facets, importing modules through the loader. */ + async resolve(plugins: readonly PluginRef[]): Promise { + const modules = plugins.filter((ref): ref is string | URL => !isFacet(ref)); + if (modules.length > 0 && this.#loader === undefined) { + throw new Error( + "loading modules needs createHost({ loader }), e.g. nodeLoader from @earendil-works/chord/node", + ); + } + const loaded = modules.length > 0 && this.#loader !== undefined ? await this.#loader(modules) : []; + return plugins.map((ref) => (isFacet(ref) ? ref : loaded.shift()!)); + } + + facetApi(serve: Serve): FacetApi { + return { + serve, + name: this.name, + root: (fn, options) => this.root(fn, options), + registry: (registry) => this.registry(registry), + }; + } + + serveInto(services: Map, service: Service): void { + if (services.has(service.contract.name)) throw new Error(`${service.contract.name} is already served`); + services.set(service.contract.name, service); + } + + startEager(service: Service): Hold { + const eager = service.options.eager; + const input = typeof eager === "object" ? eager.input : {}; + const handle = this.acquire(service.contract, input, undefined, "host"); + this.eager.set(service.contract.name, handle); + handle.ready.catch((error: unknown) => this.report(error)); + return handle; + } + + // ─── HostCore ─── + + currentEpoch(): number { + return this.current.id; + } + + hasEpoch(epoch: number): boolean { + return this.epochs.has(epoch); + } + + *retainedEpochs(): Iterable]> { + for (const [id, state] of this.epochs) yield [id, state.services]; + if (this.pending !== undefined) yield [this.pending.id, this.pending.services]; + } + + pendingEpoch(): number | undefined { + return this.pending?.id; + } + + /** The current code for a type; a type a running reload adds is served from the pending epoch. */ + service(typeName: string): Service | undefined { + return this.current.services.get(typeName) ?? this.pending?.services.get(typeName); + } + + addressOf(type: AnyContract, input: unknown, principal: Principal, at?: string, candidate = false): string { + if (at !== undefined && at !== this.name) + return remoteAddress(at, type, input, principal === this.principal ? undefined : principal); + const service = this.#bindingService(type.name, candidate); + if (service === undefined) throw unknownType(type.name, this.name); + const normalized = normalize(service.contract.input, input, `${type.name} input`); + return this.#address(service, normalized, principal); + } + + #address(service: Service, normalized: unknown, principal: Principal): string { + const key = principal === this.principal ? "host" : `p:${this.#principalKey(principal)}`; + return `${service.contract.name}\u0000${canonicalIdentity(service.contract.input, normalized)}\u0000${key}`; + } + + acquire( + type: AnyContract, + input: unknown, + options: GetOptions | undefined, + owner: HoldOwner, + from?: RecordImpl, + candidate = false, + ): Hold { + if (options?.at !== undefined && options.at !== this.name) + return this.#acquireRemote(type, input, options, owner); + const service = this.#bindingService(type.name, candidate); + if (service === undefined) throw unknownType(type.name, this.name); + bind(service.contract, service.options.accepts, type); + const principal = options?.principal ?? this.principal; + const normalized = normalize(service.contract.input, input, `${type.name} input`); + const address = this.#address(service, normalized, principal); + let record = this.records.get(address); + if (from !== undefined && record !== undefined && (record === from || reaches(record, from))) { + throw invalid("cycle", `${from.address} -> ${address} would create a hold cycle`); + } + if (record === undefined) { + if (this.gating?.has(type.name)) { + throw new ChordError("upgrade_conflict", `${type.name} is being upgraded; retry`, { reason: "gate" }); + } + const dying = this.#dying.get(address); + if (dying?.status === "disposed") this.#dying.delete(address); + record = new RecordImpl(this, { + address, + typeName: type.name, + input: normalized, + principal, + eager: + service.options.eager !== undefined && service.options.eager !== false && principal === this.principal, + }); + this.records.set(address, record); + // A new record for an address still being disposed waits for that cleanup (exclusive resources). + record.start(dying?.status === "disposing" ? dying.dispose(false) : undefined); + } + const handle = new HandleImpl(record, type, owner); + record.addHold(handle); + return handle; + } + + forget(record: RecordImpl): void { + if (this.records.get(record.address) === record) this.records.delete(record.address); + if (record.status === "disposing") this.#dying.set(record.address, record); + } + + touch(rootId: string, record: RecordImpl): void { + let touched = this.#touches.get(rootId); + if (touched === undefined) { + touched = new Set(); + this.#touches.set(rootId, touched); + } + touched.add(record); + } + + // ─── roots ─── + + async root(fn: (ctx: Ctx) => T | Promise, options: RootOptions = {}): Promise { + // The incarnation keeps root ids unique across restarts of a host with the same name. + const rootId = `${this.name}#${this.#incarnation}:${++this.#rootSeq}`; + this.#local.add(rootId); + return this.joinRoot(rootId, options.epoch ?? this.current.id, fn, options); + } + + /** + * Runs `fn` inside root `rootId` of `epoch`. A root counts as running here (for retirement) while any of its calls + * runs here. A root that started here ends when its last call here settles, even if it went out and came back + * over a link meanwhile. A foreign root (started elsewhere, arriving `from` a link) keeps its leases on records + * until every link it came over announced its end (`root_end`) or closed, so its re-entry stays grandfathered. + */ + async joinRoot( + rootId: string, + epochId: number, + fn: (ctx: Ctx) => T | Promise, + options: { readonly principal?: Principal; readonly signal?: AbortSignal; readonly from?: Connection } = {}, + ): Promise { + const epoch = this.epochs.get(epochId); + if (epoch === undefined) { + this.#local.delete(rootId); + throw new ChordError("superseded", `epoch ${epochId} is not retained by ${this.name}`); + } + if (options.from !== undefined && !this.#local.has(rootId)) { + let links = this.#foreign.get(rootId); + if (links === undefined) { + links = new Set(); + this.#foreign.set(rootId, links); + } + links.add(options.from); + } + epoch.roots.set(rootId, (epoch.roots.get(rootId) ?? 0) + 1); + const ctx = new CtxImpl(this, { + signal: options.signal ?? new AbortController().signal, + rootId, + epoch: epoch.id, + principal: options.principal ?? this.principal, + }); + try { + return await fn(ctx); + } finally { + ctx.close(); + const count = (epoch.roots.get(rootId) ?? 1) - 1; + if (count > 0) epoch.roots.set(rootId, count); + else { + epoch.roots.delete(rootId); + if (!this.#foreign.has(rootId)) this.endRoot(rootId); + this.maybeRetire(); + } + } + } + + /** Remembers that calls of `rootId` went out on `connection`, which must hear when the root ends. */ + rootSent(rootId: string, connection: Connection): void { + let links = this.#rootLinks.get(rootId); + if (links === undefined) { + links = new Set(); + this.#rootLinks.set(rootId, links); + } + links.add(connection); + } + + /** A root ended (here, or announced by its origin): end its leases and tell the links it went out on. */ + endRoot(rootId: string): void { + this.#local.delete(rootId); + this.#foreign.delete(rootId); + for (const record of this.#touches.get(rootId) ?? []) record.rootEnded(rootId); + this.#touches.delete(rootId); + for (const connection of this.#rootLinks.get(rootId) ?? []) connection.send({ t: "root_end", rootId }); + this.#rootLinks.delete(rootId); + } + + /** + * `from` announced the end of a foreign root (or closed). Honored only if the root came over `from`: a peer cannot + * end leases of roots it never carried. The root ends when no link that brought it remains. + */ + foreignRootEnded(rootId: string, from: Connection): void { + const links = this.#foreign.get(rootId); + if (links === undefined || !links.delete(from) || links.size > 0) return; + const running = [...this.epochs.values()].some((epoch) => epoch.roots.has(rootId)); + // Still running here (a call in flight): the lease ends when it settles. + if (running) this.#foreign.delete(rootId); + else this.endRoot(rootId); + } + + /** + * Retires the previous epoch once no local roots of it remain. A child also waits for its parent to retire it: + * roots span hosts, and an old root on the server may still call in. + */ + maybeRetire(): void { + const previous = this.previous; + if (previous === undefined || previous.roots.size > 0) return; + if (this.kind !== "client" && this.upstream !== undefined && previous.id > this.#retireAllowed) return; + this.retire(); + } + + allowRetire(epoch: number): void { + this.#retireAllowed = Math.max(this.#retireAllowed, epoch); + this.maybeRetire(); + } + + /** Drops the previous epoch: its behaviors and the declarations only it used. */ + retire(): void { + const previous = this.previous; + if (previous === undefined) return; + this.previous = undefined; + this.epochs.delete(previous.id); + for (const record of [...this.records.values()]) this.retireBehavior(record, previous.id); + for (const connection of this.connections) { + if (connection.role === "downstream" && connection.peer?.kind === "host") + connection.send({ t: "retire", epoch: previous.id }); + } + this.retired(); + } + + /** + * Drops a record's behavior for `epoch` and what only that behavior declared, in disposal order: resources close + * (reverse open order) before its holds are released. + */ + retireBehavior(record: RecordImpl, epoch: number): void { + const installed = record.behaviors.get(epoch); + record.behaviors.delete(epoch); + if (installed === undefined || installed.decls === record.decls) return; + const kept = record.allDecls(); + const keep = (pick: (decls: Decls) => Iterable): Set => + new Set([...kept].flatMap((decls) => [...pick(decls)])); + const holds = keep((decls) => decls.holds.values()); + const resources = keep((decls) => decls.resources.values()); + const derives = keep((decls) => decls.derives.values()); + for (const derived of installed.decls.derives.values()) if (!derives.has(derived)) derived.detach(); + const closing = [...installed.decls.resources.values()] + .filter((resource) => !resources.has(resource)) + .sort((a, b) => record.openOrder.indexOf(b) - record.openOrder.indexOf(a)); + const releasing = [...installed.decls.holds.values()].filter((hold) => !holds.has(hold)); + record.refreshSchemas(); + runAll( + [ + ...closing.map((resource) => () => resource.close(record.openOrder)), + ...releasing.map((hold) => () => hold.release()), + ], + `${record.address}: retiring epoch ${epoch} failed`, + ).catch((error: unknown) => this.report(error)); + } + + // ─── registries ─── + + registry(registry: Registry): RegistryView { + const table = (ctx?: Ctx): RegistryTable => { + const epoch = ctx === undefined ? this.current : this.epochs.get(ctx.epoch); + if (epoch === undefined) throw new ChordError("superseded", `epoch ${ctx?.epoch} is retired`); + return (epoch.tables.get(registry.name) ?? EMPTY_TABLE) as RegistryTable; + }; + return { + table, + get: (key, ctx) => table(ctx).get(key), + subscribe: (listener) => { + let set = this.#subscribers.get(registry.name); + if (set === undefined) { + set = new Set(); + this.#subscribers.set(registry.name, set); + } + const wrapped = listener as (table: RegistryTable, epoch: number) => void; + set.add(wrapped); + wrapped(table(), this.current.id); + return () => set.delete(wrapped); + }, + }; + } + + /** Builds registry tables from (record, declarations) pairs. Throws registry_conflict. */ + buildTables(entries: Iterable): Map> { + const byRegistry = new Map>(); + for (const [record, decls] of entries) { + for (const { registry, key, value } of decls.registrations.values()) { + let entries = byRegistry.get(registry.name); + if (entries === undefined) { + entries = new Map(); + byRegistry.set(registry.name, entries); + } + const existing = entries.get(key); + if (existing !== undefined && existing.record !== record && registry.conflict === "error") { + throw invalid( + "registry_conflict", + `${registry.name}: ${key} registered by ${existing.record.address} and ${record.address}`, + ); + } + entries.set(key, { value, record }); + let order = this.#order.get(registry.name); + if (order === undefined) { + order = new Map(); + this.#order.set(registry.name, order); + } + if (!order.has(key)) order.set(key, order.size); + } + } + const tables = new Map>(); + for (const [name, entries] of byRegistry) { + const order = this.#order.get(name)!; + const sorted = [...entries].sort(([a], [b]) => order.get(a)! - order.get(b)!); + tables.set(name, new Map(sorted.map(([key, entry]) => [key, entry.value]))); + } + return tables; + } + + /** + * The service a hold binds against: the current one, or for a prepare rerun (`candidate`) the running reload's + * pending one, so consumer and provider of a breaking change can reload in one batch. + */ + #bindingService(typeName: string, candidate: boolean): Service | undefined { + return (candidate ? this.pending?.services.get(typeName) : undefined) ?? this.service(typeName); + } + + /** Live records with their current declarations; records waiting for a reload's commit only with `candidates`. */ + liveDecls(candidates = false): [RecordImpl, Decls][] { + return [...this.records.values()] + .filter((record) => record.status === "live" && (candidates || !record.deferredRuntime)) + .map((record) => [record, record.decls]); + } + + checkRegistrations(record: RecordImpl, decls: Decls): void { + if (decls.registrations.size === 0) return; + this.buildTables([...this.liveDecls().filter(([other]) => other !== record), [record, decls]]); + } + + registriesChanged(including?: RecordImpl): void { + const entries = this.liveDecls().filter(([record]) => record !== including); + if (including !== undefined) entries.push([including, including.decls]); + this.installTables(this.current, this.buildTables(entries)); + } + + /** A commit failed midway: from now on this host admits no work (calls fail `unavailable/host_failed`). */ + fail(error: unknown): void { + this.failed ??= error; + this.report(error); + } + + /** Installs tables for `epoch`; subscribers hear about changed tables of the current epoch (all tables when `all`). */ + installTables(epoch: EpochState, tables: Map>, all = false): void { + const before = epoch.tables; + epoch.tables = tables; + if (epoch !== this.current) return; + for (const [name, listeners] of this.#subscribers) { + const next = tables.get(name) ?? EMPTY_TABLE; + if (!all && sameTable(before.get(name) ?? EMPTY_TABLE, next)) continue; + const current = [...listeners]; + enqueue(() => { + for (const listener of current) { + try { + listener(next, epoch.id); + } catch (error) { + this.report(error); + } + } + }); + } + } + + // ─── public API ─── + + get(type: C, input: InputOf, options?: GetOptions): Handle { + return this.acquire(type, input, options, "external").proxy as Handle; + } + + async invalidatePrincipal(match: PrincipalMatch): Promise { + const matches = typeof match === "function" ? match : (principal: Principal) => matchesPattern(match, principal); + const reason = new ChordError("gone", "principal invalidated"); + // Links acting as a matching principal are re-stamped (if the acceptor can) or closed. + for (const connection of [...this.connections]) { + if (connection.role !== "downstream" || !matches(connection.principal)) continue; + // Relays under the old principal close at once, including those still resolving their route. + connection.revoke(connection.restamp?.(connection.principal)); + } + // Patterns propagate to child hosts, which dispose their own matching records (e.g. relayed guests). + if (typeof match !== "function") { + for (const connection of this.connections) { + if (connection.role === "downstream" && connection.peer.kind === "host") { + connection.send({ t: "invalidate", pattern: match }); + } + } + } + const matching = [...this.records.values()].filter( + (record) => record.principal !== this.principal && matches(record.principal), + ); + await disposeHoldersFirst(matching, reason, "invalidatePrincipal: disposal failed"); + } + + // ─── links ─── + + accept(link: Link, options: AcceptOptions): void { + const kind = options.kind ?? "client"; + if (kind === "host" && options.name === undefined) throw new TypeError("accepting a host link needs its name"); + this.#addConnection(link, { + ...options, + role: "downstream", + principal: options.principal, + peer: { kind, name: kind === "host" ? options.name : undefined }, + }); + } + + connect(link: Link, options: LinkOptions = {}): void { + if (this.upstream?.open) this.upstream.link.close("replaced"); + const connection = this.#addConnection(link, { + ...options, + role: "upstream", + principal: undefined, + peer: { kind: "host", name: undefined }, + }); + this.upstream = connection; + // Remote handles that lost their link re-hold over the new one; state subscribers get a reset. + for (const handle of this.remotes) if (handle.binding === undefined) connection.hold(handle); + } + + #addConnection(link: Link, config: ConstructorParameters[2]): Connection { + const connection = new Connection(this, link, config); + this.connections.add(connection); + return connection; + } + + route(prefix: string, route: Route): void { + this.#routes.set(prefix, route); + } + + /** + * A child host announced its epoch. In step: usable. Behind: fenced (nothing is routed to it) and reconciled by a + * reload of its modules to this host's epoch, never by renumbering what it runs. Ahead: impossible, refused. + */ + peerHello(connection: Connection, epoch: number): void { + if (epoch === this.current.id) { + connection.markSynced(epoch); + return; + } + if (epoch > this.current.id) { + this.report(new ChordError("invalid", `${connection.name} runs epoch ${epoch}, ahead of ${this.current.id}`)); + connection.link.close("epoch ahead of parent"); + return; + } + connection.peerEpoch = epoch; + connection.fencedEpochs.add(epoch); + reconcileChild(this, connection).catch((error: unknown) => { + this.report(error); + connection.link.close("reconciliation failed"); + }); + } + + connectionClosed(connection: Connection): void { + this.connections.delete(connection); + // Foreign roots that came over this link will never announce their end over it. + for (const [rootId, links] of [...this.#foreign]) + if (links.has(connection)) this.foreignRootEnded(rootId, connection); + for (const links of this.#rootLinks.values()) links.delete(connection); + if (this.upstream === connection) { + this.upstream = undefined; + this.#childReloads.parentLost(); + } + } + + forgetRemote(handle: RemoteHandle): void { + this.remotes.delete(handle); + } + + /** A downstream host connection by its configured name (never by what a peer announced). */ + #childNamed(name: string): Connection | undefined { + return [...this.connections].find( + (connection) => + connection.role === "downstream" && connection.peer.kind === "host" && connection.peer.name === name, + ); + } + + #acquireRemote(type: AnyContract, input: unknown, options: GetOptions, owner: HoldOwner): Hold { + const at = options.at!; + if (type.scope !== "remote") throw invalid("local", `${type.name} is local and cannot be reached at ${at}`); + const principal = + options.principal === undefined || options.principal === this.principal ? undefined : options.principal; + const handle = new RemoteHandle(this, { + contract: type, + at, + input, + principal, + owner, + address: remoteAddress(at, type, input, principal), + }); + this.remotes.add(handle); + const bindWhenSynced = (connection: Promise): void => { + connection.then( + (target) => { + if (handle.status === "loading" && handle.binding === undefined) target.hold(handle); + }, + (error: unknown) => handle.gone(error), + ); + }; + const direct = this.#childNamed(at); + if (this.upstream !== undefined) this.upstream.hold(handle); + else if (direct !== undefined) bindWhenSynced(direct.whenSynced().then(() => direct)); + else { + const route = this.#routeFor(at); + if (route === undefined) handle.gone(new ChordError("unavailable", `no route to ${at}`)); + else bindWhenSynced(this.#resolve(route, at, principal ?? this.principal)); + } + return handle; + } + + #routeFor(hostName: string): Route | undefined { + let best: [string, Route] | undefined; + for (const entry of this.#routes) { + if (hostName.startsWith(entry[0]) && (best === undefined || entry[0].length > best[0].length)) best = entry; + } + return best?.[1]; + } + + /** Authorize, resolve (maybe spawn), authorize again; returns the synced connection to the host. */ + async #resolve( + route: Route, + hostName: string, + principal: Principal, + still: () => boolean = () => true, + ): Promise { + const refuse = (): ChordError => new ChordError("forbidden", `${hostName} refused for this principal`); + const revoked = (): ChordError => new ChordError("gone", "principal invalidated while resolving"); + if (!(await route.authorize(principal, hostName))) throw refuse(); + if (!still()) throw revoked(); + const link = await route.resolve(hostName); + if (!still()) throw revoked(); + if (!(await route.authorize(principal, hostName))) throw refuse(); + if (!still()) throw revoked(); + let connection = [...this.connections].find((candidate) => candidate.link === link); + connection ??= this.#addConnection(link, { + role: "downstream", + principal: route.principal?.(hostName) ?? { host: hostName }, + peer: { kind: "host", name: hostName }, + }); + await connection.whenSynced(); + if (!still()) throw revoked(); + return connection; + } + + /** + * Relays a hold from `relay.origin` to the host it names. Relaying needs a route: it is where authorization + * lives. The origin's authorization generation is re-checked after every await and right before forwarding, so a + * revocation during resolution wins. + */ + async relay(relay: RelayHold): Promise { + const at = relay.frame.at; + const route = this.#routeFor(at); + if (route === undefined) { + throw this.#childNamed(at) === undefined + ? new ChordError("unavailable", `no route to ${at}`) + : new ChordError("forbidden", `no route authorizes ${at}`); + } + const down = await this.#resolve(route, at, relay.principal, () => relay.current); + if (!relay.current) return; + relay.open(down); + } + + reload(plugins: readonly PluginRef[], options: ReloadOptions = {}): Promise { + // Client-local facet epochs never cross the parent link; only child hosts require cluster coordination. + if (this.kind !== "client" && this.upstream !== undefined) { + return Promise.reject( + new ChordError("unavailable", "reloads are coordinated by the server", { reason: "not_coordinator" }), + ); + } + const children = [...this.connections].some( + (connection) => connection.role === "downstream" && connection.peer.kind === "host", + ); + return children || options.remote !== undefined || options.remoteRemove !== undefined + ? runClusterReload(this, plugins, options) + : runReload(this, plugins, options); + } + + onReloadFrame(connection: Connection, frame: Extract): void { + this.#childReloads.handle(connection, frame); + } + + rememberRemoteModules( + remote: Readonly> | undefined, + remove: Readonly> | undefined, + ): void { + for (const prefix of new Set([...Object.keys(remote ?? {}), ...Object.keys(remove ?? {})])) { + const modules = remote?.[prefix] ?? []; + const removed = remove?.[prefix] ?? []; + this.remoteModules[prefix] = [...new Set([...(this.remoteModules[prefix] ?? []), ...modules])].filter( + (module) => !removed.includes(module), + ); + this.remoteRemoved[prefix] = [...new Set([...(this.remoteRemoved[prefix] ?? []), ...removed])].filter( + (module) => !modules.includes(module), + ); + } + } + + /** The epoch id the next reload would commit. */ + peekNextEpoch(): number { + return this.#epochSeq + 1; + } + + /** + * Disposes every record, holders before what they hold (a resource's `close` may still use the objects its record + * holds, local or remote), then closes the links. + */ + async close(): Promise { + this.eager.clear(); + const reason = new ChordError("gone", `host ${this.name} closed`); + try { + await disposeHoldersFirst([...this.records.values()], reason, "host close: disposal failed"); + } finally { + for (const connection of [...this.connections]) connection.link.close("host closed"); + } + } +} + +/** + * Force-disposes `records` in waves: a record goes once no record of the set that holds it is left, so holders are + * disposed (and their resources closed) while what they hold is still live. In-host holds are acyclic. Errors of + * every disposal are collected. + */ +async function disposeHoldersFirst(records: readonly RecordImpl[], reason: unknown, message: string): Promise { + const remaining = new Set(records); + const held = (record: RecordImpl): Set => { + const result = new Set(); + for (const decls of record.allDecls()) { + for (const hold of decls.holds.values()) if (hold.record !== undefined) result.add(hold.record); + } + return result; + }; + const holds = new Map(records.map((record) => [record, held(record)] as const)); + const errors: unknown[] = []; + while (remaining.size > 0) { + const heldByRemaining = new Set(); + for (const record of remaining) for (const target of holds.get(record) ?? []) heldByRemaining.add(target); + let wave = [...remaining].filter((record) => !heldByRemaining.has(record)); + // A cycle cannot happen in-host; dispose the rest together rather than loop. + if (wave.length === 0) wave = [...remaining]; + for (const record of wave) remaining.delete(record); + const results = await Promise.allSettled(wave.map((record) => record.dispose(true, reason))); + for (const result of results) if (result.status === "rejected") errors.push(result.reason); + } + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) throw new AggregateError(errors, message); +} + +function sameTable(a: RegistryTable, b: RegistryTable): boolean { + if (a.size !== b.size) return false; + const keys = [...a.keys()]; + return [...b].every(([key, value], index) => keys[index] === key && a.get(key) === value); +} + +const matchesPattern = (pattern: Readonly>, principal: Principal): boolean => + principal !== null && + typeof principal === "object" && + Object.entries(pattern).every(([key, value]) => deepEqual((principal as Record)[key], value)); + +/** Address of a remote hold, as the consumer sees it (the provider normalizes the input). */ +const remoteAddress = (at: string, type: AnyContract, input: unknown, principal: Principal | undefined): string => + `@${at}\u0000${type.name}\u0000${canonicalJson(input)}\u0000${principal === undefined ? "host" : canonicalJson(principal)}`; + +const unknownType = (type: string, host: string): ChordError => + new ChordError("unavailable", `${type} is not served by ${host}`, { reason: "unknown_type" }); + +/** True if `from` holds `to`, directly or transitively, through record-owned holds on this host. */ +function reaches(from: RecordImpl, to: RecordImpl): boolean { + const seen = new Set(); + const stack = [from]; + while (stack.length > 0) { + const record = stack.pop()!; + if (record === to) return true; + if (seen.has(record)) continue; + seen.add(record); + // Remote holds have no record here: cycles across hosts are avoided by convention. + for (const decls of record.allDecls()) { + for (const hold of decls.holds.values()) if (hold.record !== undefined) stack.push(hold.record); + } + } + return false; +} diff --git a/packages/chord/src/index.ts b/packages/chord/src/index.ts index 4eca4457d..83ed85a94 100644 --- a/packages/chord/src/index.ts +++ b/packages/chord/src/index.ts @@ -1,85 +1,70 @@ -/** - * Chord is a standalone application-composition runtime for agentic applications. - */ +export type { Cell } from "./cell.ts"; +export type { AcceptOptions, LinkOptions } from "./connection.ts"; +export type { Context, ContextKey } from "./context/index.ts"; export { - combineFacetLoaders, - createFacetHost, - createRemoteServiceBinding, - createStaticFacetLoader, - defineFacet, - defineService, - replicatedState, -} from "./api.ts"; + type Accepts, + type Adapter, + type AnyContract, + type BehaviorOf, + bind, + coverage, + type Description, + type Handle, + type InputOf, + type LocalContract, + local, + type MembersOf, + method, + opaque, + type RemoteContract, + remote, +} from "./contract.ts"; +export { + type CellDecl, + cell, + type DerivationDecl, + derivation, + type ListenerDecl, + listener, + type Resource, + type ResourceDecl, + type ResourceSpec, + resource, + type Subscribable, +} from "./declare.ts"; export type { Draft } from "./delta/index.ts"; -export { type CopyJsonOptions, copyJson, isJsonValue } from "./json.ts"; +export { ChordError, type ChordErrorCode, forbidden, invalid, isChordError } from "./errors.ts"; +export type { PrincipalMatch, Route } from "./host.ts"; export { - isRemoteServiceErrorCode, - REMOTE_SERVICE_ERROR_CODES, - RemoteServiceError, - type RemoteServiceErrorCode, -} from "./services/errors.ts"; + createHost, + defineFacet, + type Facet, + type FacetApi, + type FacetDefinition, + type FacetLoader, + type Host, + type HostOptions, + hostPrincipal, + type LoadOptions, + type PluginRef, + type RootOptions, +} from "./host.ts"; +export { type CopyJsonOptions, copyJson, isJsonValue, type JsonRepresentation, type JsonValue } from "./json.ts"; +export { type IpcChannel, ipcLink, type Link, memoryLinkPair, type WebSocketLike, webSocketLink } from "./link.ts"; +export type { Factory, Self, ServeOptions } from "./record.ts"; +export { defineRegistry, type Registry, type RegistryTable, type RegistryView } from "./registry.ts"; +export type { ReloadOptions, ReloadReport } from "./reload.ts"; export { - createRemoteServiceEndpoint, - type RemoteServiceEndpoint, - RemoteServiceProvider, - type ServiceUpdatePublisher, -} from "./services/provider.ts"; -export { - createServiceStateDecoder, - createServiceStateEncoder, - type ServiceStateDecoder, - type ServiceStateEncoder, -} from "./services/state-codec.ts"; -export { - createServiceCatalogueCall, - createServiceSubscribeCall, - createServiceUnsubscribeCall, - decodeServiceControlCall, - parseServiceCall, - parseServiceCatalogue, - parseServiceProviderUpdate, - parseServiceSubscriptionSnapshot, - parseWireServiceProviderUpdate, - parseWireServiceSubscriptionSnapshot, - type ServiceControlCall, - type WireServiceInstanceSnapshot, - type WireServiceMemberSnapshot, - type WireServiceProviderUpdate, - type WireServiceSubscriptionSnapshot, -} from "./services/wire.ts"; -export type { - AttachedReplicatedState, - Context, - ContextKey, - Facet, - FacetEnvironment, - FacetHost, - FacetLoader, - FacetOptions, - JsonRepresentation, - JsonValue, - LoadedFacets, - MutableReplicatedState, - RemoteServiceBinding, - RemoteServiceBindingOptions, - RemoteServiceSource, - RemoteServices, - RemoteServiceTransport, - ReplicatedState, - ReplicatedStateDelivery, - ReplicatedStateSource, - ReplicatedStateSourceAttachment, - ReplicatedStateSourceFrame, - ReplicatedStateSourceOptions, - Service, - ServiceCall, - ServiceCatalogueEntry, - ServiceInstanceAddress, - ServiceInstanceSnapshot, - ServiceMemberSnapshot, - ServiceMode, - ServiceProviderUpdate, - ServiceSpawner, - ServiceSubscription, - ServiceSubscriptionSnapshot, -} from "./types.ts"; + type AttachedReplicatedState, + type MutableReplicatedState, + type ReplicatedState, + type ReplicatedStateDelivery, + type ReplicatedStateSource, + type ReplicatedStateSourceAttachment, + type ReplicatedStateSourceFrame, + type ReplicatedStateSourceOptions, + replicatedState, +} from "./state.ts"; +export type { Ctx, GetOptions, HandleStatus, Principal, Readable, Shape, StateEvent } from "./types.ts"; +export { acceptWebSockets, type WebSocketAcceptOptions, type WebSocketServerLike } from "./websocket.ts"; +export type { Frame } from "./wire.ts"; diff --git a/packages/chord/src/json.ts b/packages/chord/src/json.ts index 4747dddd4..a45173b15 100644 --- a/packages/chord/src/json.ts +++ b/packages/chord/src/json.ts @@ -1,4 +1,19 @@ -import type { JsonValue } from "./types.ts"; +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; + +type IsAny = 0 extends 1 & T ? true : false; + +/** Strict-JSON representation of an application data type. Unknown payloads become JsonValue. */ +export type JsonRepresentation = IsAny extends true + ? JsonValue + : unknown extends T + ? JsonValue + : T extends null | boolean | number | string + ? T + : T extends readonly (infer TItem)[] + ? JsonRepresentation[] + : T extends object + ? { [TKey in keyof T]: JsonRepresentation } + : never; const DATA_DESCRIPTOR: PropertyDescriptor = { value: undefined, diff --git a/packages/chord/src/link.ts b/packages/chord/src/link.ts new file mode 100644 index 000000000..2a20415f2 --- /dev/null +++ b/packages/chord/src/link.ts @@ -0,0 +1,209 @@ +import type { Frame } from "./wire.ts"; + +/** An ordered (FIFO), bidirectional frame channel between two hosts. */ +export interface Link { + readonly closed: boolean; + send(frame: Frame): void; + /** Sets the single receiver. Frames arriving earlier are queued. */ + receive(handler: (frame: Frame) => void): void; + onClose(handler: (reason: string) => void): void; + /** Closes both ends: the peer gets a `bye`. */ + close(reason?: string): void; +} + +/** Shared plumbing: receive queue, close handlers, `bye`. Transports implement `transmit` and `shutdown`. */ +abstract class BaseLink implements Link { + #handler: ((frame: Frame) => void) | undefined; + #queue: Frame[] = []; + readonly #closeHandlers = new Set<(reason: string) => void>(); + #closed = false; + + get closed(): boolean { + return this.#closed; + } + + protected abstract transmit(frame: Frame): void; + protected abstract shutdown(): void; + + send(frame: Frame): void { + if (this.#closed) return; + this.transmit(frame); + } + + receive(handler: (frame: Frame) => void): void { + this.#handler = handler; + for (const frame of this.#queue.splice(0)) handler(frame); + } + + onClose(handler: (reason: string) => void): void { + if (this.#closed) handler("closed"); + else this.#closeHandlers.add(handler); + } + + close(reason = "closed"): void { + if (this.#closed) return; + try { + this.transmit({ t: "bye", reason }); + } catch { + // The transport may already be gone. + } + this.ended(reason); + } + + /** Delivers a parsed frame from the transport. */ + protected deliver(frame: Frame): void { + if (this.#closed) return; + if (frame.t === "bye") { + this.ended(frame.reason ?? "peer closed"); + return; + } + if (this.#handler === undefined) this.#queue.push(frame); + else this.#handler(frame); + } + + /** Marks the link closed (transport ended) and notifies once. */ + protected ended(reason: string): void { + if (this.#closed) return; + this.#closed = true; + this.#queue = []; + try { + this.shutdown(); + } finally { + for (const handler of this.#closeHandlers) handler(reason); + this.#closeHandlers.clear(); + } + } +} + +/** Parses a JSON frame; anything else is a protocol error. */ +function parseFrame(data: string): Frame { + const frame = JSON.parse(data) as Frame; + if (frame === null || typeof frame !== "object" || typeof frame.t !== "string") throw new TypeError("not a frame"); + return frame; +} + +// ─── memory ────────────────────────────────────────────────────────────────── + +export class MemoryLink extends BaseLink { + peer: MemoryLink | undefined; + #severed = false; + #pending = Promise.resolve(); + + /** Simulates a dead network: frames are dropped silently in both directions and no close is signalled. */ + sever(): void { + this.#severed = true; + if (this.peer) this.peer.#severed = true; + } + + protected transmit(frame: Frame): void { + if (this.#severed) return; + // A JSON round trip detaches the frame (and rejects what JSON would mangle silently, e.g. bigint). + const data = JSON.stringify(frame); + const peer = this.peer; + // One promise chain per direction keeps delivery FIFO and asynchronous, like a real transport. + this.#pending = this.#pending.then(() => { + if (!this.#severed && peer !== undefined) peer.deliver(parseFrame(data)); + }); + } + + protected shutdown(): void { + const peer = this.peer; + this.#pending = this.#pending.then(() => peer?.ended("peer closed")); + } +} + +/** Two connected in-memory links (tests). */ +export function memoryLinkPair(): [MemoryLink, MemoryLink] { + const a = new MemoryLink(); + const b = new MemoryLink(); + a.peer = b; + b.peer = a; + return [a, b]; +} + +// ─── Node IPC ──────────────────────────────────────────────────────────────── + +/** `process` in a forked child, or the parent's `ChildProcess`. Structural, so Chord needs no Node types here. */ +export interface IpcChannel { + readonly connected: boolean; + send(message: unknown): boolean; + on(event: "message", listener: (message: unknown) => void): unknown; + on(event: "disconnect", listener: () => void): unknown; + off(event: "message", listener: (message: unknown) => void): unknown; + off(event: "disconnect", listener: () => void): unknown; +} + +/** Chord frames on an IPC channel are wrapped as `{ chord2: frame }`; other messages (control) pass by. */ +export class IpcLink extends BaseLink { + readonly #channel: IpcChannel; + readonly #onMessage = (message: unknown): void => { + const frame = (message as { chord2?: Frame } | null)?.chord2; + if (frame !== undefined) this.deliver(frame); + }; + readonly #onDisconnect = (): void => this.ended("ipc disconnected"); + + constructor(channel: IpcChannel) { + super(); + this.#channel = channel; + channel.on("message", this.#onMessage); + channel.on("disconnect", this.#onDisconnect); + } + + protected transmit(frame: Frame): void { + if (this.#channel.connected) this.#channel.send({ chord2: frame }); + } + + protected shutdown(): void { + this.#channel.off("message", this.#onMessage); + this.#channel.off("disconnect", this.#onDisconnect); + } +} + +export const ipcLink = (channel: IpcChannel): IpcLink => new IpcLink(channel); + +// ─── WebSocket ─────────────────────────────────────────────────────────────── + +/** The WHATWG WebSocket surface: the browser `WebSocket`, Node's global `WebSocket`, and `ws` sockets. */ +export interface WebSocketLike { + readonly readyState: number; + send(data: string): void; + close(code?: number, reason?: string): void; + addEventListener(type: "message", listener: (event: { readonly data: unknown }) => void): void; + addEventListener(type: "open" | "close" | "error", listener: () => void): void; +} + +const OPEN = 1; + +export class WebSocketLink extends BaseLink { + readonly #socket: WebSocketLike; + #outbox: string[] = []; + + constructor(socket: WebSocketLike) { + super(); + this.#socket = socket; + socket.addEventListener("open", () => { + for (const data of this.#outbox.splice(0)) socket.send(data); + }); + socket.addEventListener("message", (event) => { + try { + this.deliver(parseFrame(typeof event.data === "string" ? event.data : String(event.data))); + } catch { + this.close("protocol error"); + } + }); + socket.addEventListener("close", () => this.ended("socket closed")); + } + + protected transmit(frame: Frame): void { + const data = JSON.stringify(frame); + if (this.#socket.readyState === OPEN) this.#socket.send(data); + else this.#outbox.push(data); + } + + protected shutdown(): void { + this.#outbox = []; + if (this.#socket.readyState <= OPEN) this.#socket.close(1000); + } +} + +export const webSocketLink = (socket: WebSocketLike): WebSocketLink => new WebSocketLink(socket); diff --git a/packages/chord/src/loader.ts b/packages/chord/src/loader.ts new file mode 100644 index 000000000..c46a70b68 --- /dev/null +++ b/packages/chord/src/loader.ts @@ -0,0 +1,63 @@ +// Node facet loader: `createHost({ loader: nodeLoader })`. Kept out of index.ts so the core stays browser-safe. +// +// TODO(loader): replace with an isolating, evicting loader. Every reload imports fresh module copies under a new +// query and the old copies are never evicted (they leak until the process exits). TUI plugins need isolation, which +// this loader does not provide. +import { registerHooks } from "node:module"; +import { pathToFileURL } from "node:url"; +import type { Facet, FacetDefinition, FacetLoader } from "./host.ts"; + +const PARAM = "chord2-reload"; + +/** Directory URLs (trailing slash) of loaded facet modules; their files are re-imported on reload. */ +const roots = new Set(); +let hooked = false; +let generation = 0; + +/** + * A query on the entry URL only re-imports the entry module: its relative imports would resolve to cached copies. + * This hook carries the reload query from a re-imported parent to children inside a facet module's directory. + * Modules outside (Chord itself, shared code) stay single instances. + */ +function hook(): void { + if (hooked) return; + hooked = true; + registerHooks({ + resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + const version = context.parentURL === undefined ? null : new URL(context.parentURL).searchParams.get(PARAM); + if (version === null || !result.url.startsWith("file:")) return result; + const url = new URL(result.url); + if (url.searchParams.has(PARAM) || ![...roots].some((root) => url.href.startsWith(root))) return result; + url.searchParams.set(PARAM, version); + return { ...result, url: url.href }; + }, + }); +} + +const toUrl = (module: string | URL): URL => + module instanceof URL + ? new URL(module.href) + : new URL(module.startsWith("file:") ? module : pathToFileURL(module).href); + +/** Imports fresh copies of facet modules (default export: a FacetDefinition); each facet is named by its URL. */ +export const nodeLoader: FacetLoader = async (modules) => { + hook(); + const version = String(++generation); + const facets: Facet[] = []; + for (const module of modules) { + const url = toUrl(module); + url.search = ""; + const name = url.href; + roots.add(new URL(".", url).href); + url.searchParams.set(PARAM, version); + // Owner-approved exception to the no-dynamic-import rule; see TODO(loader) above. + const loaded = (await import(url.href)) as { default?: FacetDefinition }; + const definition = loaded.default; + if (definition === undefined || typeof definition.setup !== "function") { + throw new TypeError(`${name}: default export must be a facet (defineFacet)`); + } + facets.push({ name, setup: definition.setup, uses: definition.uses }); + } + return facets; +}; diff --git a/packages/chord/src/manager.ts b/packages/chord/src/manager.ts new file mode 100644 index 000000000..cefefa6e6 --- /dev/null +++ b/packages/chord/src/manager.ts @@ -0,0 +1,416 @@ +// Host manager (Node only): runs a child host process and keeps at most one instance of it alive. +import { type ChildProcess, fork } from "node:child_process"; +import { createHash } from "node:crypto"; +import { unlinkSync } from "node:fs"; +import { rm } from "node:fs/promises"; +import { connect, createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ChordError } from "./errors.ts"; +import { type IpcChannel, type IpcLink, ipcLink } from "./link.ts"; + +/** Control messages share the IPC channel with chord frames: `{ chord2control: message }`. */ +export type ControlMessage = + | { readonly type: "init"; readonly value: unknown } + | { readonly type: "ready" } + | { readonly type: "fatal"; readonly message: string; readonly reason?: string } + | { readonly type: "shutdown" } + | { + readonly type: "status"; + readonly activeCalls?: number; + readonly viewers?: number; + readonly durableTasks?: number; + }; + +export interface Liveness { + /** Started but not ready yet. */ + readonly booting: boolean; + readonly activeCalls: number; + /** Viewers the child reports (it counts heartbeating subscribers). */ + readonly viewers: number; + readonly durableTasks: number; + /** Milliseconds since the child last reported work (calls, viewers or tasks), or since it became ready. */ + readonly idleFor: number; +} + +export interface Exit { + readonly code: number | null; + readonly signal: string | null; +} + +/** A running child: its link (for `host.accept`) and its exit. */ +export interface Instance { + readonly pid: number; + readonly link: IpcLink; + readonly exited: Promise; + send(message: ControlMessage): void; + /** Kills the child and everything it spawned. */ + killTree(): void; + onControl(listener: (message: ControlMessage) => void): void; +} + +export interface Driver { + spawn(): Instance; +} + +export interface ForkDriverOptions { + /** The child's entry module. */ + readonly entry: string; + readonly args?: readonly string[]; + readonly env?: Readonly>; + /** Node flags for the child, e.g. `["--import", sourceResolver]` (default: none). */ + readonly execArgv?: readonly string[]; + /** Sent to the child as its first control message (`childControl().init`). */ + readonly init?: unknown; + /** + * A lock name: one live process per name, across managers and processes. The child acquires it as a kernel lock + * (`childControl().lock()`, before opening its storage); a second child fails to start with `unavailable/locked`. + */ + readonly lock?: string; + /** Optional stdout/stderr tap. Streams are still mirrored to the parent's standard output. */ + readonly onOutput?: (stream: "stdout" | "stderr", chunk: Uint8Array) => void; + /** Optional application metadata tap; control messages and chord frames keep their normal handling. */ + readonly onMessage?: (message: unknown) => void; +} + +/** + * A process lock owned by the kernel for the holder's lifetime: an abstract socket (Linux, Android) or a named pipe + * (Windows) vanishes with its process, so there is no stale state to take over and no race between two would-be + * owners. Other platforms need a socket file, removed on exit; a leftover one nobody accepts on is taken over. + * The child acquires it (before opening storage), so the lock follows the process that owns the data. + */ +export async function processLock(name: string): Promise<() => Promise> { + const key = createHash("sha256").update(name).digest("hex").slice(0, 32); + const file = process.platform !== "win32" && process.platform !== "linux" && process.platform !== "android"; + const address = + process.platform === "win32" + ? `\\\\.\\pipe\\chord2-${key}` + : file + ? join(tmpdir(), `chord2-${key}.sock`) + : `\0chord2-${key}`; + const server = createServer((socket) => socket.end()); + for (;;) { + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(address, () => { + server.removeListener("error", reject); + resolve(); + }); + }); + break; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EADDRINUSE") throw error; + if (!file || (await accepting(address))) { + throw new ChordError("unavailable", `lock ${name} is held by another process`, { reason: "locked" }); + } + await rm(address, { force: true }); + } + } + const removeFile = (): void => { + try { + unlinkSync(address); + } catch { + // Already gone. + } + }; + if (file) process.once("exit", removeFile); + return () => + new Promise((resolve, reject) => { + if (file) process.removeListener("exit", removeFile); + server.close((error) => (error ? reject(error) : resolve())); + }); +} + +function accepting(address: string): Promise { + return new Promise((resolve) => { + const socket = connect(address); + socket.once("connect", () => { + socket.destroy(); + resolve(true); + }); + socket.once("error", (error: NodeJS.ErrnoException) => + resolve(error.code !== "ECONNREFUSED" && error.code !== "ENOENT"), + ); + }); +} + +const LOCK_ENV = "CHORD2_LOCK"; + +export function forkDriver(options: ForkDriverOptions): Driver { + return { + spawn() { + // Detached: the child leads a process group, so the whole tree can be killed. + const child: ChildProcess = fork(options.entry, [...(options.args ?? [])], { + env: { + ...process.env, + ...options.env, + ...(options.lock === undefined ? {} : { [LOCK_ENV]: options.lock }), + }, + execArgv: [...(options.execArgv ?? [])], + detached: true, + stdio: ["ignore", options.onOutput ? "pipe" : "inherit", options.onOutput ? "pipe" : "inherit", "ipc"], + }); + if (options.onOutput) { + for (const stream of ["stdout", "stderr"] as const) { + child[stream]?.on("data", (chunk: Buffer) => { + options.onOutput!(stream, chunk); + process[stream].write(chunk); + }); + } + } + const pid = child.pid ?? -1; + const killGroup = (): void => { + try { + process.kill(-pid, "SIGKILL"); + } catch { + // No group left. + } + }; + const exited = new Promise((resolve) => { + child.once("exit", (code, signal) => { + // However the leader ended (graceful, crash, kill), nothing it spawned outlives it. + killGroup(); + resolve({ code, signal }); + }); + }); + const listeners = new Set<(message: ControlMessage) => void>(); + child.on("message", (message) => { + options.onMessage?.(message); + const control = (message as { chord2control?: ControlMessage } | null)?.chord2control; + if (control !== undefined) for (const listener of listeners) listener(control); + }); + const send = (message: ControlMessage): void => { + if (child.connected) child.send({ chord2control: message }); + }; + if (options.init !== undefined) send({ type: "init", value: options.init }); + return { + pid, + link: ipcLink(child as unknown as IpcChannel), + exited, + send, + killTree: () => { + try { + process.kill(-pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + }, + onControl: (listener) => listeners.add(listener), + }; + }, + }; +} + +export interface HostManagerOptions { + readonly driver: Driver; + /** Evaluated every `checkInterval` while running; `false` stops the child. */ + readonly keepAlive?: (liveness: Liveness) => boolean; + readonly checkInterval?: number; + /** Milliseconds a child may take to shut down before its process tree is killed (default 5000). */ + readonly grace?: number; + /** Milliseconds to wait for `ready` (default 30000). */ + readonly readyTimeout?: number; + /** The child exited on its own (crash, fatal) or after a stop. */ + readonly onExit?: (exit: Exit) => void; +} + +export interface HostManager { + readonly status: "stopped" | "starting" | "running" | "stopping"; + readonly instance: Instance | undefined; + readonly liveness: Liveness; + /** Starts the child (or returns the running one) once it reported ready. Serialized with stop/replace. */ + start(): Promise; + /** Asks the child to shut down, then kills its process tree after the grace period. */ + stop(): Promise; + /** Stop, wait for the exit, start. */ + replace(): Promise; +} + +export function createHostManager(options: HostManagerOptions): HostManager { + let status: HostManager["status"] = "stopped"; + let instance: Instance | undefined; + let queue: Promise = Promise.resolve(); + let report = { activeCalls: 0, viewers: 0, durableTasks: 0 }; + /** When the child last became idle; undefined while it reports work. */ + let idleSince: number | undefined; + let booting = false; + let timer: ReturnType | undefined; + + const serialized = (step: () => Promise): Promise => { + const next = queue.then(step, step); + queue = next.catch(() => {}); + return next; + }; + + const liveness = (): Liveness => ({ + booting, + ...report, + idleFor: idleSince === undefined ? 0 : Date.now() - idleSince, + }); + + const doStop = async (): Promise => { + const current = instance; + if (current === undefined) { + status = "stopped"; + booting = false; + return; + } + status = "stopping"; + if (timer !== undefined) clearInterval(timer); + current.send({ type: "shutdown" }); + let graceTimer: ReturnType | undefined; + const graceful = await Promise.race([ + current.exited.then(() => true), + new Promise((resolve) => { + graceTimer = setTimeout(() => resolve(false), options.grace ?? 5000); + }), + ]); + clearTimeout(graceTimer); + if (!graceful) current.killTree(); + await current.exited; + if (instance === current) instance = undefined; + status = "stopped"; + }; + + const doStart = async (): Promise => { + if (instance !== undefined && status === "running") return instance; + status = "starting"; + booting = true; + report = { activeCalls: 0, viewers: 0, durableTasks: 0 }; + let current: Instance; + try { + current = options.driver.spawn(); + } catch (error) { + status = "stopped"; + booting = false; + throw error; + } + instance = current; + current.exited.then((exit) => { + if (instance === current) { + instance = undefined; + status = "stopped"; + if (timer !== undefined) clearInterval(timer); + } + options.onExit?.(exit); + }); + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new ChordError("unavailable", "child not ready in time")), + options.readyTimeout ?? 30_000, + ); + const settle = (fn: () => void): void => { + clearTimeout(timeout); + fn(); + }; + current.exited.then((exit) => + settle(() => + reject(new ChordError("unavailable", `child exited before ready (${exit.code ?? exit.signal})`)), + ), + ); + current.onControl((message) => { + if (message.type === "ready") settle(resolve); + else if (message.type === "fatal") + settle(() => + reject(new ChordError("unavailable", `child failed: ${message.message}`, { reason: message.reason })), + ); + else if (message.type === "status") { + report = { + activeCalls: message.activeCalls ?? 0, + viewers: message.viewers ?? 0, + durableTasks: message.durableTasks ?? 0, + }; + // The idle clock starts at the busy → idle transition, not at the last busy report. + if (report.activeCalls + report.viewers + report.durableTasks > 0) idleSince = undefined; + else idleSince ??= Date.now(); + } + }); + }).catch(async (error: unknown) => { + current.killTree(); + await current.exited; + throw error; + }); + booting = false; + if (report.activeCalls + report.viewers + report.durableTasks === 0) idleSince ??= Date.now(); + status = "running"; + if (options.keepAlive !== undefined) { + const keepAlive = options.keepAlive; + timer = setInterval(() => { + if (status === "running" && !keepAlive(liveness())) void serialized(doStop); + }, options.checkInterval ?? 1000); + timer.unref?.(); + } + return current; + }; + + return { + get status() { + return status; + }, + get instance() { + return instance; + }, + get liveness() { + return liveness(); + }, + start: () => serialized(doStart), + stop: () => serialized(doStop), + replace: () => + serialized(async () => { + await doStop(); + return doStart(); + }), + }; +} + +/** The child side of the control channel (call in the child's entry). */ +export function childControl(channel: NodeJS.Process = process): { + readonly init: Promise; + /** + * Acquires the manager's lock (if one was configured) as a kernel lock held for this process's lifetime. Call it + * before opening storage. Fails with `unavailable/locked`, also reported to the manager as a fatal start error. + */ + lock(): Promise; + ready(): void; + fatal(error: unknown): void; + report(status: { activeCalls?: number; viewers?: number; durableTasks?: number }): void; + onShutdown(handler: () => void | Promise): void; +} { + const send = (message: ControlMessage): void => { + if (channel.connected) channel.send?.({ chord2control: message }); + }; + let resolveInit: (value: unknown) => void = () => {}; + const init = new Promise((resolve) => { + resolveInit = resolve; + }); + const shutdown = new Set<() => void | Promise>(); + channel.on("message", (message: unknown) => { + const control = (message as { chord2control?: ControlMessage } | null)?.chord2control; + if (control?.type === "init") resolveInit(control.value); + if (control?.type === "shutdown") for (const handler of shutdown) void handler(); + }); + const fatal = (error: unknown): void => + send({ + type: "fatal", + message: error instanceof Error ? error.message : String(error), + ...(error instanceof ChordError && error.reason !== undefined ? { reason: error.reason } : {}), + }); + return { + init, + lock: async () => { + const name = process.env[LOCK_ENV]; + if (name === undefined) return; + try { + await processLock(name); + } catch (error) { + fatal(error); + throw error; + } + }, + ready: () => send({ type: "ready" }), + fatal, + report: (status) => send({ type: "status", ...status }), + onShutdown: (handler) => shutdown.add(handler), + }; +} diff --git a/packages/chord/src/node.ts b/packages/chord/src/node.ts index 24f2f1564..89462cfc8 100644 --- a/packages/chord/src/node.ts +++ b/packages/chord/src/node.ts @@ -1,24 +1,16 @@ -export type { - FacetBundleArtifactLoaderOptions, - FacetBundleExternalResolver, - FacetBundleLoaderOptions, -} from "./node/bundle-loader.ts"; +/** Node-only parts of Chord: child host processes, their manager, and the facet module loader. */ +export { nodeLoader } from "./loader.ts"; export { - createFacetBundleArtifactLoader, - createFacetBundleLoader, - readFacetBundleArtifact, - readFacetBundleManifest, -} from "./node/bundle-loader.ts"; -export type { - FacetBundleArtifact, - FacetBundleEntry, - FacetBundleManifest, - FacetBundlePlugin, -} from "./node/manifest.ts"; -export { - FACET_BUNDLE_ARTIFACT_FORMAT, - FACET_BUNDLE_ARTIFACT_FORMAT_VERSION, - FACET_BUNDLE_FORMAT, - FACET_BUNDLE_FORMAT_VERSION, - FACET_BUNDLE_MANIFEST_FILE, -} from "./node/manifest.ts"; + type ControlMessage, + childControl, + createHostManager, + type Driver, + type Exit, + type ForkDriverOptions, + forkDriver, + type HostManager, + type HostManagerOptions, + type Instance, + type Liveness, + processLock, +} from "./manager.ts"; diff --git a/packages/chord/src/node/bundle-loader.ts b/packages/chord/src/node/bundle-loader.ts deleted file mode 100644 index ec831c51c..000000000 --- a/packages/chord/src/node/bundle-loader.ts +++ /dev/null @@ -1,416 +0,0 @@ -import { createHash } from "node:crypto"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { createRequire, isBuiltin } from "node:module"; -import { tmpdir } from "node:os"; -import { basename, dirname, isAbsolute, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { compileFunction } from "node:vm"; -import type { Facet, FacetLoader, LoadedFacets } from "../types.ts"; -import { - FACET_BUNDLE_ARTIFACT_FORMAT, - FACET_BUNDLE_ARTIFACT_FORMAT_VERSION, - FACET_BUNDLE_FORMAT, - FACET_BUNDLE_FORMAT_VERSION, - FACET_BUNDLE_MANIFEST_FILE, - type FacetBundleArtifact, - type FacetBundleEntry, - type FacetBundleManifest, -} from "./manifest.ts"; - -export type FacetBundleExternalResolver = (specifier: string) => string | URL | undefined; - -export interface FacetBundleLoaderOptions { - readonly manifestPath: string | URL; - readonly entry: string; - /** Verify the entry's SHA-256 integrity before evaluating it. Defaults to true. */ - readonly verifyIntegrity?: boolean; - /** Resolve host-provided external imports when the bundle is outside the host's package tree. */ - readonly resolveExternal?: FacetBundleExternalResolver; -} - -export interface FacetBundleArtifactLoaderOptions { - readonly artifact: unknown; - /** Resolve host-provided external imports against the receiving application. */ - readonly resolveExternal?: FacetBundleExternalResolver; - /** Parent directory for materialized module generations. Defaults to the operating system temp directory. */ - readonly temporaryDirectory?: string; -} - -interface CommonJsModule { - exports: unknown; -} - -/** Read and validate a versioned facet bundle manifest. */ -export async function readFacetBundleManifest(path: string | URL): Promise { - const manifestPath = toFilePath(path); - let parsed: unknown; - try { - parsed = JSON.parse(await readFile(manifestPath, "utf8")); - } catch (error) { - throw new Error(`Could not read facet bundle manifest ${manifestPath}`, { cause: error }); - } - return validateManifest(parsed, manifestPath); -} - -/** Read and verify one transportable entry from a facet bundle on disk. */ -export async function readFacetBundleArtifact(options: { - readonly manifestPath: string | URL; - readonly entry: string; -}): Promise { - if (options.entry.length === 0) throw new TypeError("Facet bundle entry name must not be empty"); - const manifestPath = toFilePath(options.manifestPath); - const manifest = await readFacetBundleManifest(manifestPath); - const entry = manifest.entries[options.entry]; - if (entry === undefined) throw new Error(`Facet bundle ${manifest.plugin.id} has no entry named ${options.entry}`); - const modulePath = resolveBundleFile(manifestPath, entry.file, "entry"); - const source = await readFile(modulePath, "utf8"); - verifySource(source, entry); - const sourceMapContents = - entry.sourceMap === undefined - ? undefined - : await readFile(resolveBundleFile(manifestPath, entry.sourceMap, "source map"), "utf8"); - return Object.freeze({ - format: FACET_BUNDLE_ARTIFACT_FORMAT, - formatVersion: FACET_BUNDLE_ARTIFACT_FORMAT_VERSION, - plugin: manifest.plugin, - entryName: options.entry, - entry, - source, - ...(sourceMapContents === undefined ? {} : { sourceMapContents }), - }); -} - -/** Materialize a transported artifact and create a fresh VM-compiled CommonJS generation for each load. */ -export function createFacetBundleArtifactLoader(options: FacetBundleArtifactLoaderOptions): FacetLoader { - const artifact = validateArtifact(options.artifact); - const temporaryParent = resolve(options.temporaryDirectory ?? tmpdir()); - return { - async load(): Promise { - await mkdir(temporaryParent, { recursive: true }); - const directory = await mkdtemp(join(temporaryParent, "chord-facet-")); - try { - await materializeArtifact(directory, artifact); - const loaded = await createFacetBundleLoader({ - manifestPath: join(directory, FACET_BUNDLE_MANIFEST_FILE), - entry: artifact.entryName, - resolveExternal: options.resolveExternal, - }).load(); - let disposed = false; - return { - get facets() { - return loaded.facets; - }, - async dispose() { - if (disposed) return; - disposed = true; - const errors: unknown[] = []; - try { - await loaded.dispose(); - } catch (error) { - errors.push(error); - } - try { - await rm(directory, { force: true, recursive: true }); - } catch (error) { - errors.push(error); - } - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, "Failed to dispose facet bundle artifact"); - }, - }; - } catch (error) { - try { - await rm(directory, { force: true, recursive: true }); - } catch (cleanupError) { - throw new AggregateError([error, cleanupError], "Facet bundle artifact loading and cleanup failed"); - } - throw error; - } - }, - }; -} - -/** Create a reusable loader for one opaque entry in a facet bundle manifest. */ -export function createFacetBundleLoader(options: FacetBundleLoaderOptions): FacetLoader { - if (options.entry.length === 0) throw new TypeError("Facet bundle entry name must not be empty"); - const manifestPath = toFilePath(options.manifestPath); - return { - async load(): Promise { - const manifest = await readFacetBundleManifest(manifestPath); - const entry = manifest.entries[options.entry]; - if (entry === undefined) { - throw new Error(`Facet bundle ${manifest.plugin.id} has no entry named ${options.entry}`); - } - const modulePath = resolveBundleFile(manifestPath, entry.file, "entry"); - try { - const source = await readFile(modulePath, "utf8"); - if (options.verifyIntegrity !== false) verifySource(source, entry); - const exported = executeCommonJsModule(source, modulePath, entry.externalImports, options.resolveExternal); - let facets = facetsFromModule(exported, manifest.plugin.id, options.entry); - let disposed = false; - return { - get facets() { - return facets; - }, - async dispose() { - if (disposed) return; - disposed = true; - facets = Object.freeze([]); - }, - }; - } catch (error) { - const detail = error instanceof Error ? `: ${error.message}` : ""; - throw new Error(`Could not load facet bundle entry ${manifest.plugin.id}/${options.entry}${detail}`, { - cause: error, - }); - } - }, - }; -} - -function executeCommonJsModule( - source: string, - modulePath: string, - externalImports: readonly string[], - resolver?: FacetBundleExternalResolver, -): unknown { - const requireFromModule = createRequire(modulePath); - const externalTargets = new Map(); - for (const specifier of externalImports) { - if (!isBuiltin(specifier)) validatePackageSpecifier(specifier); - externalTargets.set(specifier, toRequireSpecifier(resolveExternalTarget(specifier, resolver))); - } - const targetFor = (specifier: string): string => { - const target = externalTargets.get(specifier); - if (target === undefined) throw new Error(`Facet bundle required undeclared external import: ${specifier}`); - return target; - }; - const requireExternal = Object.assign((specifier: string): unknown => requireFromModule(targetFor(specifier)), { - resolve: (specifier: string): string => requireFromModule.resolve(targetFor(specifier)), - }); - const commonJsModule: CommonJsModule = { exports: Object.create(null) }; - const compiled = compileFunction(source, ["exports", "require", "module", "__filename", "__dirname"], { - filename: modulePath, - }); - Reflect.apply(compiled, commonJsModule.exports, [ - commonJsModule.exports, - requireExternal, - commonJsModule, - modulePath, - dirname(modulePath), - ]); - return commonJsModule.exports; -} - -function toRequireSpecifier(target: string): string { - if (isAbsolute(target) || isBuiltin(target)) return target; - let url: URL; - try { - url = new URL(target); - } catch { - return target; - } - if (url.protocol === "file:") return fileURLToPath(url); - if (url.protocol === "node:") return url.href; - throw new Error(`Facet bundle external target must be a file, package, or built-in module: ${target}`); -} - -function toFilePath(path: string | URL): string { - if (typeof path === "string") return resolve(path); - if (path.protocol !== "file:") throw new TypeError(`Facet bundle manifest must be a file URL, not ${path.protocol}`); - return fileURLToPath(path); -} - -function resolveBundleFile(manifestPath: string, file: string, label: string): string { - if (file.length === 0 || isAbsolute(file) || basename(file) !== file || file === "." || file === "..") { - throw new Error(`Facet bundle ${label} must be a filename relative to its manifest`); - } - return resolve(dirname(manifestPath), file); -} - -function verifySource(source: string, entry: FacetBundleEntry): void { - const expected = parseIntegrity(entry.integrity); - const actual = createHash("sha256").update(source, "utf8").digest("base64"); - if (actual !== expected) throw new Error(`Facet bundle integrity check failed for ${entry.file}`); -} - -function parseIntegrity(integrity: string): string { - const prefix = "sha256-"; - if (!integrity.startsWith(prefix) || integrity.length === prefix.length) { - throw new Error("Facet bundle entry has an invalid SHA-256 integrity value"); - } - return integrity.slice(prefix.length); -} - -function facetsFromModule(imported: unknown, pluginId: string, entryName: string): readonly Facet[] { - if (!isRecord(imported)) throw new Error(`Facet bundle entry ${pluginId}/${entryName} did not export a module`); - const exported = imported.default; - const candidates: readonly unknown[] = Array.isArray(exported) ? (exported as readonly unknown[]) : [exported]; - if (candidates.length === 0) { - throw new Error(`Facet bundle entry ${pluginId}/${entryName} exported no facets`); - } - const facets: Facet[] = []; - for (const candidate of candidates) { - if (!isRecord(candidate) || typeof candidate.id !== "string" || candidate.id.length === 0) { - throw new Error(`Facet bundle entry ${pluginId}/${entryName} has a facet with an invalid ID`); - } - if (typeof candidate.setup !== "function") { - throw new Error(`Facet bundle entry ${pluginId}/${entryName} facet ${candidate.id} has no setup function`); - } - facets.push(candidate as unknown as Facet); - } - const ids = facets.map(({ id }) => id); - if (new Set(ids).size !== ids.length) { - throw new Error(`Facet bundle entry ${pluginId}/${entryName} exports duplicate facet IDs`); - } - return Object.freeze(facets); -} - -function validateArtifact(value: unknown): FacetBundleArtifact { - if ( - !isRecord(value) || - value.format !== FACET_BUNDLE_ARTIFACT_FORMAT || - value.formatVersion !== FACET_BUNDLE_ARTIFACT_FORMAT_VERSION || - typeof value.entryName !== "string" || - value.entryName.length === 0 || - typeof value.source !== "string" - ) { - throw new Error("Invalid facet bundle artifact"); - } - const manifest = validateManifest( - { - format: FACET_BUNDLE_FORMAT, - formatVersion: FACET_BUNDLE_FORMAT_VERSION, - plugin: value.plugin, - entries: { [value.entryName]: value.entry }, - }, - "facet bundle artifact", - ); - const entry = manifest.entries[value.entryName]!; - if (entry.sourceMap === undefined) { - if (value.sourceMapContents !== undefined) { - throw new Error("Facet bundle artifact has source map contents without a source map"); - } - } else if (typeof value.sourceMapContents !== "string") { - throw new Error("Facet bundle artifact is missing its source map contents"); - } - verifySource(value.source, entry); - return Object.freeze({ - format: FACET_BUNDLE_ARTIFACT_FORMAT, - formatVersion: FACET_BUNDLE_ARTIFACT_FORMAT_VERSION, - plugin: manifest.plugin, - entryName: value.entryName, - entry, - source: value.source, - ...(value.sourceMapContents === undefined ? {} : { sourceMapContents: value.sourceMapContents }), - }); -} - -async function materializeArtifact(directory: string, artifact: FacetBundleArtifact): Promise { - const manifest: FacetBundleManifest = { - format: FACET_BUNDLE_FORMAT, - formatVersion: FACET_BUNDLE_FORMAT_VERSION, - plugin: artifact.plugin, - entries: { [artifact.entryName]: artifact.entry }, - }; - await Promise.all([ - writeFile(join(directory, artifact.entry.file), artifact.source, "utf8"), - writeFile(join(directory, FACET_BUNDLE_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`, "utf8"), - ...(artifact.entry.sourceMap === undefined - ? [] - : [writeFile(join(directory, artifact.entry.sourceMap), artifact.sourceMapContents!, "utf8")]), - ]); -} - -function resolveExternalTarget(specifier: string, resolver?: FacetBundleExternalResolver): string { - const resolved = resolver?.(specifier); - if (resolved !== undefined) return typeof resolved === "string" ? resolved : resolved.href; - const extension = import.meta.url.endsWith(".ts") ? "ts" : "js"; - if (specifier === "@earendil-works/chord") return new URL(`../index.${extension}`, import.meta.url).href; - if (specifier === "@earendil-works/chord/context") { - return new URL(`../context/index.${extension}`, import.meta.url).href; - } - if (specifier === "@earendil-works/chord/node") return new URL(`../node.${extension}`, import.meta.url).href; - return import.meta.resolve(specifier); -} - -function validatePackageSpecifier(specifier: string): void { - const segments = specifier.split("/"); - const packageParts = specifier.startsWith("@") ? 2 : 1; - if ( - specifier.startsWith(".") || - specifier.startsWith("/") || - specifier.startsWith("#") || - specifier.includes(":") || - specifier.includes("\\") || - segments.length < packageParts || - segments.some((part) => part.length === 0 || part === "." || part === "..") - ) { - throw new Error(`Facet bundle artifact has an unsupported external import: ${specifier}`); - } -} - -function validateManifest(value: unknown, path: string): FacetBundleManifest { - if (!isRecord(value) || value.format !== FACET_BUNDLE_FORMAT) { - throw new Error(`Invalid facet bundle manifest format in ${path}`); - } - if (value.formatVersion !== FACET_BUNDLE_FORMAT_VERSION) { - throw new Error(`Unsupported facet bundle manifest version in ${path}: ${String(value.formatVersion)}`); - } - if (!isRecord(value.plugin) || typeof value.plugin.id !== "string" || value.plugin.id.length === 0) { - throw new Error(`Facet bundle manifest has an invalid plugin identity in ${path}`); - } - if ( - value.plugin.version !== undefined && - (typeof value.plugin.version !== "string" || value.plugin.version.length === 0) - ) { - throw new Error(`Facet bundle manifest has an invalid plugin version in ${path}`); - } - if (!isRecord(value.entries) || Object.keys(value.entries).length === 0) { - throw new Error(`Facet bundle manifest has no entries in ${path}`); - } - const entries: Record = {}; - for (const [name, candidate] of Object.entries(value.entries)) { - if (name.length === 0 || !isRecord(candidate)) { - throw new Error(`Facet bundle manifest has an invalid entry in ${path}`); - } - if (typeof candidate.file !== "string") throw new Error(`Facet bundle entry ${name} has no file`); - resolveBundleFile(path, candidate.file, `entry ${name}`); - if (typeof candidate.integrity !== "string") throw new Error(`Facet bundle entry ${name} has no integrity`); - parseIntegrity(candidate.integrity); - if ( - !Array.isArray(candidate.externalImports) || - candidate.externalImports.some((item) => typeof item !== "string") - ) { - throw new Error(`Facet bundle entry ${name} has invalid external imports`); - } - const externalImports = Object.freeze([...(candidate.externalImports as readonly string[])]); - if (new Set(externalImports).size !== externalImports.length) { - throw new Error(`Facet bundle entry ${name} has duplicate external imports`); - } - if (candidate.sourceMap !== undefined) { - if (typeof candidate.sourceMap !== "string") - throw new Error(`Facet bundle entry ${name} has an invalid source map`); - resolveBundleFile(path, candidate.sourceMap, `entry ${name} source map`); - } - entries[name] = Object.freeze({ - file: candidate.file, - integrity: candidate.integrity, - externalImports, - ...(candidate.sourceMap === undefined ? {} : { sourceMap: candidate.sourceMap }), - }); - } - return Object.freeze({ - format: FACET_BUNDLE_FORMAT, - formatVersion: FACET_BUNDLE_FORMAT_VERSION, - plugin: Object.freeze({ - id: value.plugin.id, - ...(value.plugin.version === undefined ? {} : { version: value.plugin.version }), - }), - entries: Object.freeze(entries), - }); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/packages/chord/src/node/bundle.ts b/packages/chord/src/node/bundle.ts deleted file mode 100644 index c7057bb14..000000000 --- a/packages/chord/src/node/bundle.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { createHash, randomUUID } from "node:crypto"; -import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; -import { basename, dirname, extname, join, relative, resolve, sep } from "node:path"; -import { type BuildOptions, build, type Message } from "esbuild"; -import { - FACET_BUNDLE_FORMAT, - FACET_BUNDLE_FORMAT_VERSION, - FACET_BUNDLE_MANIFEST_FILE, - type FacetBundleEntry, - type FacetBundleManifest, -} from "./manifest.ts"; - -export type FacetBundlePlatform = "node" | "browser" | "neutral"; - -export interface BundleFacetsOptions { - readonly plugin: { - readonly id: string; - readonly version?: string; - }; - /** Opaque application-selected entry names mapped to TypeScript or JavaScript source files. */ - readonly entries: Readonly>; - readonly outdir: string; - readonly workingDirectory?: string; - /** Additional package imports intentionally left for the loading application to resolve. */ - readonly external?: readonly string[]; - readonly sourceMap?: boolean; - readonly minify?: boolean; - readonly define?: Readonly>; - readonly platform?: FacetBundlePlatform; - readonly target?: string | readonly string[]; -} - -export interface BundleFacetsResult { - readonly manifest: FacetBundleManifest; - readonly manifestPath: string; -} - -/** Bundle each opaque facet entry into an independent content-addressed CommonJS file. */ -export async function bundleFacets(options: BundleFacetsOptions): Promise { - validateOptions(options); - const workingDirectory = resolve(options.workingDirectory ?? process.cwd()); - const outputDirectory = resolve(workingDirectory, options.outdir); - const outputParent = dirname(outputDirectory); - await mkdir(outputParent, { recursive: true }); - const temporaryDirectory = join(outputParent, `.${basename(outputDirectory)}.tmp-${randomUUID()}`); - await mkdir(temporaryDirectory); - try { - const entries: Record = {}; - for (const [entryName, source] of Object.entries(options.entries).sort(([left], [right]) => - left.localeCompare(right), - )) { - entries[entryName] = await bundleEntry({ - entryName, - source: resolve(workingDirectory, source), - temporaryDirectory, - workingDirectory, - options, - }); - } - const manifest: FacetBundleManifest = Object.freeze({ - format: FACET_BUNDLE_FORMAT, - formatVersion: FACET_BUNDLE_FORMAT_VERSION, - plugin: Object.freeze({ - id: options.plugin.id, - ...(options.plugin.version === undefined ? {} : { version: options.plugin.version }), - }), - entries: Object.freeze(entries), - }); - await writeFile(join(temporaryDirectory, FACET_BUNDLE_MANIFEST_FILE), `${JSON.stringify(manifest, null, 2)}\n`); - await replaceDirectory(temporaryDirectory, outputDirectory); - return Object.freeze({ - manifest, - manifestPath: join(outputDirectory, FACET_BUNDLE_MANIFEST_FILE), - }); - } catch (error) { - await rm(temporaryDirectory, { force: true, recursive: true }); - throw error; - } -} - -async function bundleEntry(input: { - readonly entryName: string; - readonly source: string; - readonly temporaryDirectory: string; - readonly workingDirectory: string; - readonly options: BundleFacetsOptions; -}): Promise { - const entryPrefix = `facet-${shortHash(input.entryName)}`; - const requestedPlatform = input.options.platform ?? "node"; - const buildOptions: BuildOptions = { - absWorkingDir: input.workingDirectory, - banner: { js: '"use strict";' }, - bundle: true, - define: input.options.define, - entryNames: `${entryPrefix}-[hash]`, - entryPoints: [input.source], - external: [...new Set(["@earendil-works/chord", "@earendil-works/chord/*", ...(input.options.external ?? [])])], - format: "cjs", - legalComments: "none", - logLevel: "silent", - metafile: true, - minify: input.options.minify ?? false, - outdir: input.temporaryDirectory, - outExtension: { ".js": ".cjs" }, - platform: requestedPlatform, - sourcemap: input.options.sourceMap === true ? "external" : false, - supported: { "dynamic-import": false }, - target: - input.options.target === undefined - ? requestedPlatform === "node" - ? "node22.19" - : "es2022" - : typeof input.options.target === "string" - ? input.options.target - : [...input.options.target], - write: true, - }; - let result: Awaited>; - try { - result = await build(buildOptions); - } catch (error) { - const diagnostics = esbuildMessages(error); - const detail = diagnostics.length === 0 ? "" : `\n${diagnostics.join("\n")}`; - throw new Error(`Could not bundle facet entry ${input.entryName}${detail}`, { cause: error }); - } - const metafile = result.metafile; - if (metafile === undefined) throw new Error(`Facet entry ${input.entryName} produced no build metadata`); - const outputs = Object.entries(metafile.outputs).filter( - ([path, output]) => output.entryPoint !== undefined && extname(path) === ".cjs", - ); - if (outputs.length !== 1) - throw new Error(`Facet entry ${input.entryName} did not produce exactly one JavaScript file`); - const [outputPath, metadata] = outputs[0]!; - const absoluteOutputPath = resolve(input.workingDirectory, outputPath); - const file = relative(input.temporaryDirectory, absoluteOutputPath); - if (file.length === 0 || file.startsWith(`..${sep}`) || basename(file) !== file) { - throw new Error(`Facet entry ${input.entryName} produced an invalid output path`); - } - const sourceMap = input.options.sourceMap === true ? `${file}.map` : undefined; - const allowedOutputs = new Set([absoluteOutputPath]); - if (sourceMap !== undefined) allowedOutputs.add(join(input.temporaryDirectory, sourceMap)); - const unexpectedOutputs = Object.keys(metafile.outputs) - .map((path) => resolve(input.workingDirectory, path)) - .filter((path) => !allowedOutputs.has(path)); - if (unexpectedOutputs.length > 0) { - throw new Error(`Facet entry ${input.entryName} produced files other than its JavaScript bundle and source map`); - } - const contents = await readFile(absoluteOutputPath); - if (sourceMap !== undefined) await stat(join(input.temporaryDirectory, sourceMap)); - return Object.freeze({ - file, - integrity: `sha256-${createHash("sha256").update(contents).digest("base64")}`, - externalImports: Object.freeze( - [...new Set(metadata.imports.filter(({ external }) => external).map(({ path }) => path))].sort(), - ), - ...(sourceMap === undefined ? {} : { sourceMap }), - }); -} - -function validateOptions(options: BundleFacetsOptions): void { - if (options.plugin.id.length === 0) throw new TypeError("Facet bundle plugin ID must not be empty"); - if (options.plugin.version !== undefined && options.plugin.version.length === 0) { - throw new TypeError("Facet bundle plugin version must not be empty"); - } - const entries = Object.entries(options.entries); - if (entries.length === 0) throw new TypeError("Facet bundle must contain at least one entry"); - for (const [name, source] of entries) { - if (name.length === 0) throw new TypeError("Facet bundle entry name must not be empty"); - if (source.length === 0) throw new TypeError(`Facet bundle entry ${name} must have a source path`); - } - for (const external of options.external ?? []) { - if (external.length === 0) throw new TypeError("Facet bundle external import must not be empty"); - } -} - -async function replaceDirectory(temporaryDirectory: string, outputDirectory: string): Promise { - const backupDirectory = `${outputDirectory}.old-${randomUUID()}`; - let movedExisting = false; - try { - await rename(outputDirectory, backupDirectory); - movedExisting = true; - } catch (error) { - if (!isMissingPath(error)) throw error; - } - try { - await rename(temporaryDirectory, outputDirectory); - } catch (error) { - if (movedExisting) await rename(backupDirectory, outputDirectory); - throw error; - } - if (movedExisting) await rm(backupDirectory, { force: true, recursive: true }); -} - -function shortHash(value: string): string { - return createHash("sha256").update(value).digest("hex").slice(0, 12); -} - -function esbuildMessages(error: unknown): string[] { - if (!isRecord(error) || !Array.isArray(error.errors)) return []; - return (error.errors as readonly unknown[]).flatMap((candidate) => - isEsbuildMessage(candidate) ? [formatEsbuildMessage(candidate)] : [], - ); -} - -function formatEsbuildMessage(message: Message): string { - const location = message.location; - return location === null - ? message.text - : `${location.file}:${location.line}:${location.column + 1}: ${message.text}`; -} - -function isEsbuildMessage(value: unknown): value is Message { - return isRecord(value) && typeof value.text === "string" && (value.location === null || isRecord(value.location)); -} - -function isMissingPath(error: unknown): boolean { - return isRecord(error) && error.code === "ENOENT"; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/packages/chord/src/node/manifest.ts b/packages/chord/src/node/manifest.ts deleted file mode 100644 index e9d471e9f..000000000 --- a/packages/chord/src/node/manifest.ts +++ /dev/null @@ -1,39 +0,0 @@ -export const FACET_BUNDLE_FORMAT = "chord.facet-bundle"; -export const FACET_BUNDLE_FORMAT_VERSION = 2; -export const FACET_BUNDLE_MANIFEST_FILE = "chord-facets.json"; -export const FACET_BUNDLE_ARTIFACT_FORMAT = "chord.facet-bundle-artifact"; -export const FACET_BUNDLE_ARTIFACT_FORMAT_VERSION = 2; - -export interface FacetBundleEntry { - /** Content-addressed CommonJS filename relative to the manifest. */ - readonly file: string; - /** SHA-256 subresource-integrity value for the JavaScript file. */ - readonly integrity: string; - /** Imports intentionally left for the loading application to resolve. */ - readonly externalImports: readonly string[]; - /** Source map filename relative to the manifest, when emitted. */ - readonly sourceMap?: string; -} - -export interface FacetBundleManifest { - readonly format: typeof FACET_BUNDLE_FORMAT; - readonly formatVersion: typeof FACET_BUNDLE_FORMAT_VERSION; - readonly plugin: FacetBundlePlugin; - readonly entries: Readonly>; -} - -export interface FacetBundlePlugin { - readonly id: string; - readonly version?: string; -} - -/** One self-contained manifest entry suitable for storage or transport to another Node host. */ -export interface FacetBundleArtifact { - readonly format: typeof FACET_BUNDLE_ARTIFACT_FORMAT; - readonly formatVersion: typeof FACET_BUNDLE_ARTIFACT_FORMAT_VERSION; - readonly plugin: FacetBundlePlugin; - readonly entryName: string; - readonly entry: FacetBundleEntry; - readonly source: string; - readonly sourceMapContents?: string; -} diff --git a/packages/chord/src/node/package.ts b/packages/chord/src/node/package.ts deleted file mode 100644 index 943374e39..000000000 --- a/packages/chord/src/node/package.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { readFile, realpath, stat } from "node:fs/promises"; -import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; -import { type BundleFacetsResult, bundleFacets } from "./bundle.ts"; - -export interface BundleFacetPackageOptions { - /** Plugin package directory or its package.json path. */ - readonly packagePath: string; - readonly outdir: string; - /** Application conventions applied when the corresponding source file exists. */ - readonly defaultFacets?: Readonly>; -} - -export interface BundleFacetPackageResult extends BundleFacetsResult { - readonly packageDirectory: string; - readonly packageJsonPath: string; -} - -interface FacetPackageMetadata { - readonly packageDirectory: string; - readonly packageJsonPath: string; - readonly name: string; - readonly version: string; - readonly peerDependencies: readonly string[]; - readonly configuredFacets: Readonly>; - readonly external: readonly string[]; - readonly sourceMap: boolean; -} - -/** Build a plugin package using package.json metadata and application-provided facet conventions. */ -export async function bundleFacetPackage(options: BundleFacetPackageOptions): Promise { - const metadata = await readFacetPackageMetadata(options.packagePath); - const entries = await resolveFacetEntries(metadata, options.defaultFacets ?? {}); - const external = [...metadata.peerDependencies, ...metadata.external].flatMap((specifier) => [ - specifier, - `${specifier}/*`, - ]); - const result = await bundleFacets({ - plugin: { id: metadata.name, version: metadata.version }, - entries, - outdir: options.outdir, - workingDirectory: metadata.packageDirectory, - external, - sourceMap: metadata.sourceMap, - }); - return Object.freeze({ - ...result, - packageDirectory: metadata.packageDirectory, - packageJsonPath: metadata.packageJsonPath, - }); -} - -async function readFacetPackageMetadata(packagePath: string): Promise { - if (packagePath.length === 0) throw new TypeError("Facet package path must not be empty"); - const candidate = resolve(packagePath); - let packageDirectory: string; - let packageJsonPath: string; - let candidateStats: Awaited>; - try { - candidateStats = await stat(candidate); - } catch (error) { - throw new Error(`Could not access facet package ${candidate}`, { cause: error }); - } - if (candidateStats.isDirectory()) { - packageDirectory = await realpath(candidate); - packageJsonPath = join(packageDirectory, "package.json"); - } else if (candidateStats.isFile() && basename(candidate) === "package.json") { - packageJsonPath = await realpath(candidate); - packageDirectory = dirname(packageJsonPath); - } else { - throw new Error(`Facet package path must name a directory or package.json: ${candidate}`); - } - - let parsed: unknown; - try { - parsed = JSON.parse(await readFile(packageJsonPath, "utf8")); - } catch (error) { - throw new Error(`Could not read facet package metadata ${packageJsonPath}`, { cause: error }); - } - if (!isRecord(parsed)) throw new Error(`Facet package metadata must be an object: ${packageJsonPath}`); - if (typeof parsed.name !== "string" || parsed.name.length === 0) { - throw new Error(`Facet package must have a non-empty name: ${packageJsonPath}`); - } - if (typeof parsed.version !== "string" || parsed.version.length === 0) { - throw new Error(`Facet package must have a non-empty version: ${packageJsonPath}`); - } - const peerDependencies = parsePeerDependencies(parsed.peerDependencies, packageJsonPath); - const chord = parseChordConfiguration(parsed.chord, packageJsonPath); - return { - packageDirectory, - packageJsonPath, - name: parsed.name, - version: parsed.version, - peerDependencies, - configuredFacets: chord.facets, - external: chord.external, - sourceMap: chord.sourceMap, - }; -} - -function parsePeerDependencies(value: unknown, packageJsonPath: string): readonly string[] { - if (value === undefined) return []; - if ( - !isRecord(value) || - Object.entries(value).some(([name, version]) => name.length === 0 || typeof version !== "string") - ) { - throw new Error(`Facet package has invalid peerDependencies: ${packageJsonPath}`); - } - return Object.freeze(Object.keys(value).sort()); -} - -function parseChordConfiguration( - value: unknown, - packageJsonPath: string, -): { - readonly facets: Readonly>; - readonly external: readonly string[]; - readonly sourceMap: boolean; -} { - if (value === undefined) return { facets: Object.freeze({}), external: Object.freeze([]), sourceMap: true }; - if (!isRecord(value)) throw new Error(`Facet package chord configuration must be an object: ${packageJsonPath}`); - if (Object.keys(value).some((key) => key !== "facets" && key !== "external" && key !== "sourceMap")) { - throw new Error(`Facet package chord configuration has an unknown field: ${packageJsonPath}`); - } - const facets: Record = {}; - if (value.facets !== undefined) { - if (!isRecord(value.facets)) { - throw new Error(`Facet package chord.facets must be an object: ${packageJsonPath}`); - } - for (const [name, source] of Object.entries(value.facets)) { - if (name.length === 0 || (typeof source !== "string" && source !== false) || source === "") { - throw new Error(`Facet package has an invalid chord.facets entry: ${packageJsonPath}`); - } - facets[name] = source; - } - } - let external: readonly string[] = []; - if (value.external !== undefined) { - if ( - !Array.isArray(value.external) || - value.external.some((specifier) => typeof specifier !== "string" || specifier.length === 0) - ) { - throw new Error(`Facet package chord.external must contain non-empty strings: ${packageJsonPath}`); - } - external = Object.freeze([...new Set(value.external as readonly string[])].sort()); - } - if (value.sourceMap !== undefined && typeof value.sourceMap !== "boolean") { - throw new Error(`Facet package chord.sourceMap must be a boolean: ${packageJsonPath}`); - } - return { - facets: Object.freeze(facets), - external, - sourceMap: value.sourceMap ?? true, - }; -} - -async function resolveFacetEntries( - metadata: FacetPackageMetadata, - defaultFacets: Readonly>, -): Promise>> { - const entries: Record = {}; - for (const [name, source] of Object.entries(defaultFacets)) { - validateFacetMapping(name, source, "default"); - const path = resolvePackageEntry(metadata.packageDirectory, source, name); - try { - const entryStats = await stat(path); - if (!entryStats.isFile()) throw new Error(`Default facet entry ${name} is not a file: ${path}`); - const canonicalPath = await realpath(path); - validateCanonicalPackageEntry(metadata.packageDirectory, canonicalPath, name); - entries[name] = canonicalPath; - } catch (error) { - if (isMissingPath(error)) continue; - throw error; - } - } - for (const [name, source] of Object.entries(metadata.configuredFacets)) { - if (source === false) { - delete entries[name]; - continue; - } - validateFacetMapping(name, source, "configured"); - const path = resolvePackageEntry(metadata.packageDirectory, source, name); - let entryStats: Awaited>; - try { - entryStats = await stat(path); - } catch (error) { - throw new Error(`Could not access configured facet entry ${name}: ${path}`, { cause: error }); - } - if (!entryStats.isFile()) throw new Error(`Configured facet entry ${name} is not a file: ${path}`); - const canonicalPath = await realpath(path); - validateCanonicalPackageEntry(metadata.packageDirectory, canonicalPath, name); - entries[name] = canonicalPath; - } - if (Object.keys(entries).length === 0) { - throw new Error(`Facet package ${metadata.name} has no configured or conventional facet entries`); - } - return Object.freeze(entries); -} - -function validateFacetMapping(name: string, source: string, kind: string): void { - if (name.length === 0) throw new Error(`Facet package ${kind} entry name must not be empty`); - if (source.length === 0) throw new Error(`Facet package ${kind} entry ${name} must have a source path`); -} - -function resolvePackageEntry(packageDirectory: string, source: string, name: string): string { - if (isAbsolute(source)) throw new Error(`Facet package entry ${name} must be relative to the package directory`); - const path = resolve(packageDirectory, source); - const relativePath = relative(packageDirectory, path); - if ( - relativePath.length === 0 || - relativePath === ".." || - relativePath.startsWith(`..${sep}`) || - isAbsolute(relativePath) - ) { - throw new Error(`Facet package entry ${name} escapes the package directory`); - } - return path; -} - -function validateCanonicalPackageEntry(packageDirectory: string, path: string, name: string): void { - const relativePath = relative(packageDirectory, path); - if (relativePath === ".." || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { - throw new Error(`Facet package entry ${name} resolves outside the package directory`); - } -} - -function isMissingPath(error: unknown): boolean { - return isRecord(error) && error.code === "ENOENT"; -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} diff --git a/packages/chord/src/publish.ts b/packages/chord/src/publish.ts new file mode 100644 index 000000000..b74720ae8 --- /dev/null +++ b/packages/chord/src/publish.ts @@ -0,0 +1,39 @@ +// One FIFO queue for every observer notification (state events, shape pushes, registry tables, listener +// invocations). +// +// Problem: notifying observers synchronously from inside a write lets a re-entrant write overtake the write being +// published. Example: subscriber A reacts to n = 1 by writing n = 2; subscriber B then sees 2 before 1. +// Solution: a publication first adopts every revision it makes (cells and derivations), then enqueues its +// notifications. Notifications run in enqueue order, so every observer sees revisions in adoption order. +// `holdPublications` defers delivery until a multi-step change (a reload commit) is complete. + +const queue: (() => void)[] = []; +let held = 0; +let draining = false; + +/** Runs `task` after everything enqueued before it. Tasks must not throw (report instead). */ +export function enqueue(task: () => void): void { + queue.push(task); + drain(); +} + +/** Delivers nothing until `fn` returns, then everything it enqueued, in order. */ +export function holdPublications(fn: () => T): T { + held++; + try { + return fn(); + } finally { + held--; + drain(); + } +} + +function drain(): void { + if (draining || held > 0) return; + draining = true; + try { + for (let task = queue.shift(); task !== undefined; task = queue.shift()) task(); + } finally { + draining = false; + } +} diff --git a/packages/chord/src/record.ts b/packages/chord/src/record.ts new file mode 100644 index 000000000..7fbe89999 --- /dev/null +++ b/packages/chord/src/record.ts @@ -0,0 +1,1516 @@ +import type { TSchema } from "typebox"; +import { Value } from "typebox/value"; +import { Cell, Derived, isSource, notReady } from "./cell.ts"; +import { + type Accepts, + type Adapter, + type AnyContract, + type BehaviorOf, + bind, + type Handle, + type InputOf, + LOCAL_VARIANT, + type MemberDescription, +} from "./contract.ts"; +import { + type CellDecl, + type Cells, + type DerivationDecl, + type Derivations, + declKind, + type ListenerDecl, + type Resource, + type ResourceDecl, + type ResourceSpec, + type Resources, + type Subscribable, +} from "./declare.ts"; +import { asChordError, ChordError, invalid, runAll } from "./errors.ts"; +import { enqueue } from "./publish.ts"; +import type { Registration, Registry } from "./registry.ts"; +import { assertValid } from "./schema.ts"; +import type { Ctx, Exposed, GetOptions, HandleStatus, Principal, Readable, Shape, StateEvent } from "./types.ts"; + +const noop = (): void => {}; + +// ─── Services ──────────────────────────────────────────────────────────────── + +export interface ServeOptions { + /** + * State version: the meaning of the record's cells. At a bump, `migrate[name](old, fromVersion)` maps a cell's value; + * the record may be several versions behind. + */ + readonly state?: { + readonly version: number; + readonly migrate?: Readonly unknown>>; + }; + /** Milliseconds a record stays alive after its last hold is released. */ + readonly linger?: number; + /** Host-held record created with this input (true: `{}` normalized). Only eager records may register. */ + readonly eager?: boolean | { readonly input: InputOf }; + /** Adapters for consumers built against older contract versions. */ + readonly accepts?: Accepts; +} + +export type Factory = (input: InputOf, principal: Principal, self: Self) => BehaviorOf; +type AnyFactory = (input: unknown, principal: Principal, self: Self) => unknown; + +export interface Service { + readonly contract: AnyContract; + readonly factory: AnyFactory; + readonly options: ServeOptions; + readonly facet: string; + readonly stateVersion: number; +} + +export const makeService = ( + contract: C, + factory: Factory, + options: ServeOptions, + facet: string, +): Service => ({ + contract, + factory: factory as unknown as AnyFactory, + options: options as ServeOptions, + facet, + stateVersion: options.state?.version ?? 1, +}); + +// ─── Declarations ──────────────────────────────────────────────────────────── + +export interface Self { + /** Cells by name. Existing names keep their cell (and value) across reloads. */ + state>>>(cells: D): Cells; + /** A record-owned hold, identified by its address. */ + get(type: C, input: InputOf, options?: GetOptions): Handle; + resources>>>(resources: D): Resources; + listen(listeners: Readonly>): void; + tasks(tasks: Readonly Promise>>): void; + derive>>>(derivations: D): Derivations; + /** Eager records only. Keys are the registry's own keys. */ + register(registry: Registry, entries: Readonly>): void; + /** + * A member backed by one of this record's resources, for local contracts: handles read the resource's value once + * the record is live (`undefined` before). Typically an `opaque` member: `{ harness: self.expose(res.durable) }`. + */ + expose(resource: Resource): Exposed; +} + +/** Runtime form of `Exposed`. */ +class ExposedImpl { + readonly resource: ResourceImpl; + constructor(resource: ResourceImpl) { + this.resource = resource; + } +} + +export class ResourceImpl implements Resource { + readonly key: string; + readonly spec: ResourceSpec; + readonly deps: readonly (Hold | ResourceImpl)[]; + /** `unavailable`: a close/reopen failed during a reload rollback; the value is gone for good. */ + state: "closed" | "opening" | "open" | "unavailable" = "closed"; + #value: T | undefined; + + constructor(key: string, spec: ResourceSpec, deps: readonly (Hold | ResourceImpl)[]) { + this.key = key; + this.spec = spec; + this.deps = deps; + } + + get value(): T { + if (this.state === "unavailable") + throw new ChordError("unavailable", `resource ${this.key} could not be restored`); + if (this.state !== "open") throw notReady(`resource ${this.key}`); + return this.#value as T; + } + + get exclusive(): boolean { + return this.spec.exclusive === true; + } + + /** + * Opens the resource. An open that completes after `signal` aborted (rollback, abandonment) closes the value it + * produced at once: nobody owns it any more. + */ + async open(signal: AbortSignal, order: ResourceImpl[]): Promise { + if (this.state !== "closed") return; + this.state = "opening"; + let value: T; + try { + value = await this.spec.open({ signal }); + } catch (error) { + this.state = "closed"; + throw error; + } + if (signal.aborted) { + this.state = "closed"; + await this.spec.close?.(value); + throw new ChordError("aborted", `resource ${this.key} opened after abort`); + } + this.#value = value; + this.state = "open"; + order.push(this); + } + + async close(order: ResourceImpl[]): Promise { + if (this.state !== "open") return; + this.state = "closed"; + const index = order.indexOf(this); + if (index !== -1) order.splice(index, 1); + const value = this.#value as T; + this.#value = undefined; + await this.spec.close?.(value); + } + + markUnavailable(): void { + this.state = "unavailable"; + this.#value = undefined; + } +} + +/** + * What `self.resources` returns: the resource, except that `.value` throws `not_ready` while any factory runs, even + * if the resource is open (a reload's prepare rerun must not touch the native resource the old code uses). + */ +class ResourceFacade implements Resource { + readonly #resource: ResourceImpl; + readonly #host: HostCore; + + constructor(resource: ResourceImpl, host: HostCore) { + this.#resource = resource; + this.#host = host; + facades.set(this, resource as ResourceImpl); + } + + get key(): string { + return this.#resource.key; + } + + get value(): T { + if (this.#host.inFactory > 0) throw notReady(`resource ${this.key} (inside a factory)`); + return this.#resource.value; + } +} + +const facades = new WeakMap(); +const resourceOf = (value: unknown): ResourceImpl | undefined => + value instanceof ResourceImpl ? value : value !== null && typeof value === "object" ? facades.get(value) : undefined; + +type TaskFn = (ctx: Ctx) => Promise; + +/** Everything one factory run declared, by kind and key. */ +export class Decls { + readonly cells = new Map>(); + readonly cellSchemas = new Map(); + readonly holds = new Map(); + readonly resources = new Map(); + readonly listeners = new Map(); + readonly tasks = new Map(); + readonly derives = new Map>(); + readonly deriveFns = new Map unknown>(); + readonly registrations = new Map(); +} + +/** One factory run: its behavior, declarations and the objects it created (not reused from the previous run). */ +export interface Run { + readonly behavior: Record; + readonly decls: Decls; + readonly created: { + readonly holds: Hold[]; + readonly resources: ResourceImpl[]; + readonly derives: Derived[]; + }; +} + +export interface Installed { + readonly epoch: number; + readonly service: Service; + readonly behavior: Record; + readonly decls: Decls; +} + +// ─── Host interface the runtime pieces use ─────────────────────────────────── + +export interface HostCore { + readonly name: string; + readonly dev: boolean; + readonly principal: Principal; + inFactory: number; + /** Set when a commit failed midway: the host admits no more work (terminal). */ + readonly failed: unknown; + report(error: unknown): void; + currentEpoch(): number; + hasEpoch(epoch: number): boolean; + retainedEpochs(): Iterable]>; + pendingEpoch(): number | undefined; + service(typeName: string): Service | undefined; + acquire( + type: AnyContract, + input: unknown, + options: GetOptions | undefined, + owner: HoldOwner, + from?: RecordImpl, + candidate?: boolean, + ): Hold; + addressOf(type: AnyContract, input: unknown, principal: Principal, at?: string, candidate?: boolean): string; + root(fn: (ctx: Ctx) => T | Promise, options?: { epoch?: number; signal?: AbortSignal }): Promise; + touch(rootId: string, record: RecordImpl): void; + forget(record: RecordImpl): void; + checkRegistrations(record: RecordImpl, decls: Decls): void; + /** Rebuilds the current registry tables from live records (plus `including`, which is finishing activation). */ + registriesChanged(including?: RecordImpl): void; +} + +// ─── Contexts ──────────────────────────────────────────────────────────────── + +export class CtxImpl implements Ctx { + readonly host: HostCore; + readonly signal: AbortSignal; + readonly rootId: string; + readonly epoch: number; + readonly principal: Principal; + /** Code holders (tasks): every call they make starts a fresh root. */ + readonly holder: boolean; + readonly #holds: Hold[] = []; + #closed = false; + + /** True after the call or root that received this ctx settled: it may no longer start work. */ + get closed(): boolean { + return this.#closed; + } + + /** Throws unless this ctx may start work on `host` now (not expired, not aborted, not from another host). */ + check(host: HostCore): void { + if (this.host !== host) throw invalid("foreign_ctx", "ctx belongs to another host"); + if (this.#closed) throw new ChordError("aborted", "ctx expired: its call already settled", { reason: "expired" }); + if (this.signal.aborted) throw new ChordError("aborted", "ctx aborted", { cause: this.signal.reason }); + } + + constructor( + host: HostCore, + fields: { signal: AbortSignal; rootId: string; epoch: number; principal: Principal; holder?: boolean }, + ) { + this.host = host; + this.signal = fields.signal; + this.rootId = fields.rootId; + this.epoch = fields.epoch; + this.principal = fields.principal; + this.holder = fields.holder ?? false; + } + + get(type: C, input: InputOf, options?: GetOptions): Handle { + this.check(this.host); + const handle = this.host.acquire(type, input, { principal: this.principal, ...options }, "call"); + this.#holds.push(handle); + return handle.proxy as Handle; + } + + /** + * A context for a call into `record`: same root and epoch, its own call-owned holds, aborted on forced disposal. + * Code inside an object acts as the host principal. + */ + child(record: RecordImpl): CtxImpl { + return new CtxImpl(this.host, { + signal: AbortSignal.any([this.signal, record.callAbort.signal]), + rootId: this.rootId, + epoch: this.epoch, + principal: this.host.principal, + }); + } + + close(): void { + this.#closed = true; + for (const handle of this.#holds.splice(0)) handle.release(); + } +} + +export async function raceSignal(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) throw new ChordError("aborted", "call aborted", { cause: signal.reason }); + let onAbort = noop; + const aborted = new Promise((_, reject) => { + onAbort = () => reject(new ChordError("aborted", "call aborted", { cause: signal.reason })); + signal.addEventListener("abort", onAbort, { once: true }); + }); + try { + return await Promise.race([promise, aborted]); + } finally { + signal.removeEventListener("abort", onAbort); + } +} + +// ─── Handles ───────────────────────────────────────────────────────────────── + +/** Who a hold belongs to. `connection`: a peer host over a link; `client`: a browser (not a reload participant). */ +export type HoldOwner = "record" | "call" | "host" | "external" | "connection" | "client"; + +/** A claim on an object: a local `HandleImpl`, or a `RemoteHandle` on another host (no record here). */ +export interface Hold { + readonly address: string; + readonly contract: AnyContract; + readonly status: HandleStatus; + readonly ready: Promise; + readonly proxy: object; + readonly record: RecordImpl | undefined; + release(): void; +} + +export const HANDLE_BASE: ReadonlySet = new Set([ + "contract", + "status", + "ready", + "kind", + "shape", + "error", + "onShape", + "onStatus", + "release", +]); + +export type StatusListener = (status: HandleStatus, error: unknown) => void; + +/** Queues a status notification (after the change that caused it is complete, like every publication). */ +export function notifyStatus( + listeners: ReadonlySet, + status: HandleStatus, + error: unknown, + report: (error: unknown) => void, +): void { + if (listeners.size === 0) return; + const current = [...listeners]; + enqueue(() => { + for (const listener of current) { + try { + listener(status, error); + } catch (thrown) { + report(thrown); + } + } + }); +} + +/** Proxy → hold, for `deps: [handle]` and links. */ +export const holdsByProxy = new WeakMap(); +export const handleOf = (value: unknown): Hold | undefined => + value !== null && typeof value === "object" ? holdsByProxy.get(value) : undefined; + +/** + * A handle's state member: a stable readable that follows whatever source the current behavior exposes under that + * name. `undefined` until the handle is live (or while the variant lacks the member). Rebinding (reload, variant) + * delivers a reset. Upstream is subscribed only while someone listens. + */ +class MemberReadable implements Readable { + readonly #handle: HandleImpl; + readonly #name: string; + readonly #values = new Set<(value: unknown) => void>(); + readonly #events = new Set<(event: StateEvent) => void>(); + #source: Readable | undefined; + #upstream = noop; + /** The value listeners last saw; a first upstream snapshot equal to it is not re-announced. */ + #seen: unknown; + + constructor(handle: HandleImpl, name: string) { + this.#handle = handle; + this.#name = name; + } + + get value(): unknown { + return this.#current()?.value; + } + + subscribe(listener: (value: unknown) => void): () => void { + this.#values.add(listener); + this.#seen = this.value; + this.#safe(() => listener(this.#seen)); + this.bind(); + return () => this.#remove(this.#values, listener); + } + + events(listener: (event: StateEvent) => void): () => void { + this.#events.add(listener); + this.#seen = this.value; + this.#safe(() => listener({ type: "snap", value: this.#seen })); + this.bind(); + return () => this.#remove(this.#events, listener); + } + + /** (Re)subscribes upstream to the current source; a changed source is announced as a reset. */ + bind(): void { + const source = this.#values.size + this.#events.size > 0 ? this.#current() : undefined; + if (source === this.#source) return; + this.#upstream(); + this.#upstream = noop; + this.#source = source; + if (source === undefined) return; + let first = true; + this.#upstream = source.events((event) => { + const isFirst = first; + first = false; + if (isFirst && event.value === this.#seen) return; + this.#seen = event.value; + const translated: StateEvent = isFirst ? { type: "reset", value: event.value } : event; + const events = [...this.#events]; + const values = [...this.#values]; + // Queued: a rebind during a commit must not run observers before the commit completes. + enqueue(() => { + for (const listener of events) if (this.#events.has(listener)) this.#safe(() => listener(translated)); + for (const listener of values) if (this.#values.has(listener)) this.#safe(() => listener(event.value)); + }); + }); + } + + stop(): void { + this.#values.clear(); + this.#events.clear(); + this.bind(); + } + + #current(): Readable | undefined { + const view = this.#handle.currentView(); + if (view === undefined || !this.#handle.inVariant(view, this.#name)) return undefined; + const value = view[this.#name]; + // Cells and derivations, or any readable an `accepts` adapter maps them to. + return isSource(value) || isReadableLike(value) ? (value as Readable) : undefined; + } + + #remove(set: Set, listener: T): void { + set.delete(listener); + this.bind(); + } + + #safe(fn: () => void): void { + try { + fn(); + } catch (error) { + this.#handle.record.host.report(error); + } + } +} + +/** One hold on a record. Exposed through a Proxy so members bind late to the current behavior. */ +export class HandleImpl { + readonly record: RecordImpl; + readonly contract: AnyContract; + readonly owner: HoldOwner; + readonly address: string; + readonly proxy: object; + readonly ready: Promise; + status: HandleStatus; + error: unknown; + readonly shapeListeners = new Set<(shape: Shape) => void>(); + /** Internal: notified when the handle becomes gone (links forward it). */ + readonly goneListeners = new Set<(error: unknown) => void>(); + readonly readables = new Map(); + readonly bindings = new Map(); + readonly views = new WeakMap, Record>(); + readonly callers = new Map Promise>(); + + constructor(record: RecordImpl, contract: AnyContract, owner: HoldOwner) { + this.record = record; + this.contract = contract; + this.owner = owner; + this.address = record.address; + this.status = record.status === "live" ? "live" : "loading"; + this.ready = record.ready.then(() => { + if (this.status === "gone") throw this.error; + if (this.status === "released") throw new ChordError("released", `handle ${this.address} released`); + }); + this.ready.catch(noop); + this.proxy = new Proxy(this, { + get: (target, property) => { + if (HANDLE_BASE.has(property)) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + if (typeof property !== "string" || property === "then") return undefined; + return target.member(property); + }, + has: (target, property) => HANDLE_BASE.has(property) || target.member(String(property)) !== undefined, + }); + holdsByProxy.set(this.proxy, this); + } + + get kind(): string | undefined { + const view = this.currentView(); + return view === undefined ? undefined : kindOf(view); + } + + get shape(): Shape | undefined { + const installed = this.status === "live" ? this.record.currentInstalled() : undefined; + const view = installed === undefined ? undefined : this.#safeView(installed); + if (installed === undefined || view === undefined) return undefined; + return { + kind: kindOf(view), + members: Object.keys(view).filter((key) => key !== "kind"), + contract: installed.service.contract.fingerprint, + epoch: installed.epoch, + }; + } + + onShape(listener: (shape: Shape) => void): () => void { + this.shapeListeners.add(listener); + return () => this.shapeListeners.delete(listener); + } + + onStatus(listener: StatusListener): () => void { + this.#statusListeners.add(listener); + return () => this.#statusListeners.delete(listener); + } + + readonly #statusListeners = new Set(); + + #statusChanged(): void { + notifyStatus(this.#statusListeners, this.status, this.error, (error) => this.record.host.report(error)); + } + + release(): void { + if (this.status === "released" || this.status === "gone") return; + this.status = "released"; + this.#stopReadables(); + this.#statusChanged(); + this.record.release(this); + } + + // ─── internal ─── + + live(): void { + if (this.status !== "loading") return; + this.status = "live"; + this.#statusChanged(); + this.shapeChanged(); + } + + gone(error: unknown): void { + if (this.status === "released" || this.status === "gone") return; + this.status = "gone"; + this.error = error; + this.#stopReadables(); + this.#statusChanged(); + for (const listener of [...this.goneListeners]) listener(error); + } + + shapeChanged(): void { + for (const readable of this.readables.values()) readable.bind(); + const shape = this.shape; + if (shape === undefined) return; + const listeners = [...this.shapeListeners]; + enqueue(() => { + for (const listener of listeners) { + try { + listener(shape); + } catch (error) { + this.record.host.report(error); + } + } + }); + } + + terminalError(): unknown { + return this.status === "gone" ? this.error : new ChordError("released", `handle ${this.address} released`); + } + + /** The behavior as this consumer sees it (through an accepts adapter when bound to another version). */ + viewOf(installed: Installed): Record { + let adapter = this.bindings.get(installed.service); + if (!this.bindings.has(installed.service)) { + adapter = bind(installed.service.contract, installed.service.options.accepts, this.contract); + this.bindings.set(installed.service, adapter); + } + if (adapter === undefined) return installed.behavior; + let view = this.views.get(installed.behavior); + if (view === undefined) { + view = adapter(installed.behavior); + this.views.set(installed.behavior, view); + } + return view; + } + + currentView(): Record | undefined { + if (this.status !== "live") return undefined; + const installed = this.record.currentInstalled(); + return installed === undefined ? undefined : this.#safeView(installed); + } + + /** The view, or undefined when this consumer no longer binds (reported; calls reject with the bind error). */ + #safeView(installed: Installed): Record | undefined { + try { + return this.viewOf(installed); + } catch (error) { + this.record.host.report(error); + return undefined; + } + } + + describe(kind: string | undefined, member: string): MemberDescription | undefined { + const variants = this.contract.description.variants; + if (kind !== undefined) return variants[kind]?.[member]; + for (const members of Object.values(variants)) if (members[member] !== undefined) return members[member]; + return undefined; + } + + /** Remote consumers see only the members their contract lists for the current variant. */ + inVariant(view: Record, name: string): boolean { + return this.contract.scope !== "remote" || this.describe(kindOf(view), name) !== undefined; + } + + member(name: string): unknown { + const declared = this.describe(undefined, name); + const view = this.currentView(); + const value = view?.[name]; + if (declared?.kind === "method" || (declared === undefined && typeof value === "function")) { + return this.caller(name); + } + if (declared?.kind === "state" || (declared === undefined && (isSource(value) || isReadableLike(value)))) { + let readable = this.readables.get(name); + if (readable === undefined) { + readable = new MemberReadable(this, name); + this.readables.set(name, readable); + } + return readable; + } + if (view === undefined || !this.inVariant(view, name)) return undefined; + return value instanceof ExposedImpl ? value.resource.value : value; + } + + caller(name: string): (args: unknown, ctx?: Ctx) => Promise { + let caller = this.callers.get(name); + if (caller === undefined) { + caller = (args, ctx) => { + if (this.record.host.inFactory > 0) { + throw new ChordError("not_ready", `${this.address}.${name} called inside a factory`); + } + if (ctx !== undefined && !(ctx instanceof CtxImpl)) { + return Promise.reject(invalid("foreign_ctx", "ctx must come from a Chord host")); + } + if (ctx === undefined || ctx.holder) { + // A code holder (task) starts a fresh root per call, cancelled with the holder. + try { + ctx?.check(this.record.host); + } catch (error) { + return Promise.reject(error); + } + return this.record.host.root((root) => this.record.call(this, name, args, root as CtxImpl), { + signal: ctx?.signal, + }); + } + return this.record.call(this, name, args, ctx); + }; + this.callers.set(name, caller); + } + return caller; + } + + #stopReadables(): void { + for (const readable of this.readables.values()) readable.stop(); + } +} + +export const kindOf = (behavior: Record): string | undefined => + typeof behavior.kind === "string" ? behavior.kind : undefined; + +// ─── Records ───────────────────────────────────────────────────────────────── + +export type RecordStatus = "activating" | "live" | "failed" | "disposing" | "disposed"; + +interface Gate { + readonly grandfathered: Set; + readonly queue: { readonly key: string; readonly event: unknown }[]; + drained: () => void; +} + +interface Subscription { + /** What the listener was declared on: the resource (not its facade), or the source itself. */ + readonly origin: unknown; + readonly select: ListenerDecl["select"]; + unsubscribe: () => void; +} + +/** + * Two `select` functions are the same if they are identical or have the same source text: an inline arrow is a new + * object on every factory run, but its code is what matters. (A select whose behavior depends on captured values + * that change between runs is not detected; such a listener keeps its old subscription until its source changes.) + */ +const sameSelect = (a: ListenerDecl["select"], b: ListenerDecl["select"]): boolean => + a === b || (a !== undefined && b !== undefined && String(a) === String(b)); + +/** Listener subscriptions made ahead of a commit: `activate` switches over, `discard` drops them. */ +export interface StagedListeners { + activate(): void; + discard(): void; +} + +interface RunningTask { + readonly controller: AbortController; + readonly done: Promise; +} + +export interface RecordIdentity { + readonly address: string; + readonly typeName: string; + readonly input: unknown; + readonly principal: Principal; + readonly eager: boolean; +} + +export class RecordImpl { + readonly host: HostCore; + readonly address: string; + readonly typeName: string; + readonly input: unknown; + readonly principal: Principal; + readonly eager: boolean; + status: RecordStatus = "activating"; + ready: Promise = Promise.resolve(); + service: Service | undefined; + stateVersion = 0; + /** Calls carrying an older epoch are rejected upgrade_conflict (state bump or exclusive resource replaced). */ + minEpoch = 0; + decls = new Decls(); + /** Declarations of a factory run that has not been installed yet (activation or reload candidate). */ + readonly pending = new Set(); + readonly behaviors = new Map(); + readonly holds = new Set(); + readonly leases = new Map(); + readonly openOrder: ResourceImpl[] = []; + readonly callAbort = new AbortController(); + readonly activationAbort = new AbortController(); + gate: Gate | undefined; + /** Served only by a running reload's pending epoch: listeners, tasks and registrations start at its commit. */ + deferredRuntime = false; + readonly #subscriptions = new Map(); + readonly #invocations = new Set>(); + #tasks = new Map(); + readonly #stopped: Promise[] = []; + #linger: ReturnType | undefined; + #disposal: Promise | undefined; + + constructor(host: HostCore, identity: RecordIdentity) { + this.host = host; + this.address = identity.address; + this.typeName = identity.typeName; + this.input = identity.input; + this.principal = identity.principal; + this.eager = identity.eager; + } + + start(after: Promise | undefined): void { + this.ready = this.#activate(after).then( + () => { + for (const handle of this.holds) handle.live(); + this.#maybeIdle(); + }, + (cause: unknown) => { + if (this.status === "activating") this.status = "failed"; + this.host.forget(this); + // A ChordError (forbidden, a dependency's error, ...) keeps its code; anything else is `unavailable`. + const error = asChordError(cause, `activating ${this.address}`); + for (const handle of this.holds) handle.gone(error); + throw error; + }, + ); + this.ready.catch(noop); + } + + /** + * Activation as one transaction: factory, opens, then a synchronous finalization (registry check and publication, + * behaviors, runtime). Every failure, including one during finalization or a release while listeners start, + * unwinds everything this activation did. + */ + async #activate(after: Promise | undefined): Promise { + await after?.catch(noop); + const signal = this.activationAbort.signal; + const abandoned = (): ChordError => new ChordError("aborted", `${this.address} abandoned`); + if (signal.aborted) throw abandoned(); + const service = this.host.service(this.typeName); + if (service === undefined) + throw new ChordError("unavailable", `${this.typeName} is not served`, { reason: "unknown_type" }); + this.service = service; + this.stateVersion = service.stateVersion; + const run = this.run(service, undefined); + try { + await this.open(run.created.resources, run.created.holds, signal); + if (signal.aborted) throw abandoned(); + this.host.checkRegistrations(this, run.decls); + this.decls = run.decls; + for (const [retained, services] of this.host.retainedEpochs()) { + if (services.get(this.typeName) === service) { + this.behaviors.set(retained, { epoch: retained, service, behavior: run.behavior, decls: run.decls }); + } + } + const pending = this.host.pendingEpoch(); + if (!this.behaviors.has(this.host.currentEpoch()) && (pending === undefined || !this.behaviors.has(pending))) { + throw new ChordError("superseded", `${this.address}: code changed during activation`); + } + for (const derived of run.created.derives) derived.attach(); + this.refreshSchemas(); + // Served only by a running reload's pending epoch: new code must not run (listeners, tasks, registrations) + // before that reload commits. + this.deferredRuntime = !this.behaviors.has(this.host.currentEpoch()); + if (!this.deferredRuntime) { + if (run.decls.registrations.size > 0) this.host.registriesChanged(this); + this.startRuntime(); + } + // A listener starting up may have released the last hold synchronously: no resurrection. + if (signal.aborted) throw abandoned(); + this.status = "live"; + } catch (error) { + // Invocations started during finalization (listener roots) end before their resources close. + this.callAbort.abort(error); + this.stopListeners(); + this.stopTasks(); + const registered = this.decls.registrations.size > 0 && !this.deferredRuntime; + this.behaviors.clear(); + this.decls = new Decls(); + if (registered) this.host.registriesChanged(); + await Promise.allSettled([...this.#invocations, ...this.#stopped.splice(0)]); + await this.unwind(run).catch((cleanup: unknown) => this.host.report(cleanup)); + throw error; + } finally { + this.pending.delete(run.decls); + } + } + + /** + * Runs the factory synchronously. With `previous`, runs in recording mode for a reload prepare. The run's + * declarations stay in `pending` (visible to the cycle check) until the caller installs or discards them. + */ + run(service: Service, previous: Decls | undefined): Run { + const self = new SelfImpl(this, previous); + const discard = (run: Run): void => { + this.pending.delete(run.decls); + void this.unwind(run).catch((cleanup: unknown) => this.host.report(cleanup)); + }; + this.pending.add(self.run(undefined).decls); + this.host.inFactory++; + let behavior: unknown; + try { + behavior = service.factory(this.input, this.principal, self); + } catch (error) { + this.host.inFactory--; + discard(self.run(undefined)); + throw error; + } + this.host.inFactory--; + const run = self.run(behavior); + try { + if (behavior instanceof Promise) throw invalid("behavior", `${this.typeName}: factories must be synchronous`); + validateBehavior(service, run.behavior); + } catch (error) { + discard(run); + throw error; + } + return run; + } + + /** Opens resources (dependencies first) and waits for holds; settles all before failing. */ + async open(resources: readonly ResourceImpl[], holds: readonly Hold[], signal: AbortSignal): Promise { + const opening = new Map>(); + const openOne = (resource: ResourceImpl): Promise => { + let promise = opening.get(resource); + if (promise === undefined) { + promise = (async () => { + await Promise.all( + resource.deps.map((dep) => + !(dep instanceof ResourceImpl) ? dep.ready : resources.includes(dep) ? openOne(dep) : undefined, + ), + ); + await resource.open(signal, this.openOrder); + })(); + opening.set(resource, promise); + } + return promise; + }; + const results = await Promise.allSettled([...resources.map(openOne), ...holds.map((hold) => hold.ready)]); + const failure = results.find((result) => result.status === "rejected"); + if (failure !== undefined) throw failure.reason; + } + + /** Undoes the objects a run created: closes its resources (reverse order) and releases its holds. */ + async unwind(run: Run): Promise { + for (const derived of run.created.derives) derived.detach(); + const resources = [...run.created.resources].sort( + (a, b) => this.openOrder.indexOf(b) - this.openOrder.indexOf(a), + ); + await runAll( + [ + ...resources.map((resource) => () => resource.close(this.openOrder)), + ...run.created.holds.map((hold) => () => hold.release()), + ], + `${this.address}: cleanup failed`, + ); + } + + currentInstalled(): Installed | undefined { + return this.behaviors.get(this.host.currentEpoch()) ?? [...this.behaviors.values()].at(-1); + } + + /** Every state source exposed as a remote state member gets that member's schema (checked per write in dev mode). */ + refreshSchemas(): void { + const schemas = new Map, TSchema[]>(); + for (const installed of this.behaviors.values()) { + for (const [source, list] of memberSchemas(installed.service, installed.behavior)) { + schemas.set(source, [...(schemas.get(source) ?? []), ...list]); + } + } + for (const decls of this.allDecls()) { + for (const source of [...decls.cells.values(), ...decls.derives.values()]) + source.schemas = schemas.get(source) ?? []; + } + } + + allDecls(): Set { + const result = new Set([this.decls, ...this.pending]); + for (const installed of this.behaviors.values()) result.add(installed.decls); + return result; + } + + // ─── runtime: listeners and tasks of the current declarations ─── + + /** The resource a listener is declared on, or its source itself. */ + #origin(decl: ListenerDecl): unknown { + return resourceOf(decl.source) ?? decl.source; + } + + /** Resolves a listener's subscribable: `select(resource value)`, `select(source)` or the source itself. */ + #listenSource(key: string, decl: ListenerDecl): Subscribable { + const resource = resourceOf(decl.source); + const value = resource === undefined ? decl.source : resource.value; + const source = (decl.select === undefined ? value : decl.select(value as never)) as Subscribable; + if (typeof source?.subscribe !== "function") + throw invalid("behavior", `listener ${key}: source has no subscribe()`); + return source; + } + + /** + * Subscribes the listeners of `decls` without delivering: events are buffered until `activate`. A listener declared + * on the same resource (or source) with the same `select` keeps its subscription without calling `select` again + * (only its function is swapped, by name), so sources that accept a single subscriber survive reloads. Throws, with + * nothing left subscribed, if any source fails. + */ + stageListeners(decls: Decls): StagedListeners { + const fresh = new Map(); + const kept = new Set(); + try { + for (const [key, decl] of decls.listeners) { + const origin = this.#origin(decl); + const previous = this.#subscriptions.get(key); + if (previous !== undefined && previous.origin === origin && sameSelect(previous.select, decl.select)) { + kept.add(key); + continue; + } + const source = this.#listenSource(key, decl); + const entry: Subscription & { buffer: unknown[] | undefined } = { + origin, + select: decl.select, + unsubscribe: noop, + buffer: [], + }; + fresh.set(key, entry); + entry.unsubscribe = source.subscribe((event) => { + if (entry.buffer !== undefined) entry.buffer.push(event); + else this.#deliver(key, event); + }); + } + } catch (error) { + for (const entry of fresh.values()) entry.unsubscribe(); + throw error; + } + return { + activate: () => { + for (const [key, subscription] of this.#subscriptions) { + if (kept.has(key)) continue; + subscription.unsubscribe(); + this.#subscriptions.delete(key); + } + for (const [key, entry] of fresh) { + this.#subscriptions.set(key, entry); + const buffered = entry.buffer ?? []; + entry.buffer = undefined; + for (const event of buffered) this.#deliver(key, event); + } + }, + discard: () => { + for (const entry of fresh.values()) entry.unsubscribe(); + }, + }; + } + + startRuntime(): void { + this.stageListeners(this.decls).activate(); + this.startTasks(); + } + + startTasks(): void { + for (const key of this.decls.tasks.keys()) this.startTask(key, this.host.currentEpoch()); + } + + stopListeners(): void { + for (const subscription of this.#subscriptions.values()) subscription.unsubscribe(); + this.#subscriptions.clear(); + } + + startTask(key: string, epoch: number): void { + const fn = this.decls.tasks.get(key); + if (fn === undefined) return; + const controller = new AbortController(); + const ctx = new CtxImpl(this.host, { + signal: AbortSignal.any([controller.signal, this.callAbort.signal]), + rootId: `task:${this.address}:${key}`, + epoch, + principal: this.host.principal, + holder: true, + }); + const done = (async () => { + try { + await fn(ctx); + } catch (error) { + if (!ctx.signal.aborted) this.host.report(error); + } finally { + ctx.close(); + if (this.#tasks.get(key)?.controller === controller) this.#tasks.delete(key); + } + })(); + this.#tasks.set(key, { controller, done }); + } + + /** Aborts running tasks; their completion is awaited at disposal. */ + stopTasks(): void { + for (const task of this.#tasks.values()) { + task.controller.abort(new ChordError("aborted", "task stopped")); + this.#stopped.push(task.done); + } + this.#tasks.clear(); + } + + /** + * Delivers one listener event, in publication order. The invocation is a root that holds a lease on this record + * (a reload gate waits for it like for any other root that touched the record), counts as an invocation (disposal + * waits for it) and is cancelled by forced disposal. + */ + #deliver(key: string, event: unknown): void { + enqueue(() => { + if (this.gate !== undefined) { + this.gate.queue.push({ key, event }); + return; + } + const decl = this.decls.listeners.get(key); + if (decl === undefined || (this.status !== "live" && this.status !== "activating")) return; + const invocation = this.host.root((root) => { + this.lease(root); + const child = (root as CtxImpl).child(this); + return this.#invoke(async () => { + try { + await decl.fn(event, child); + } finally { + child.close(); + } + }); + }); + invocation.catch((error: unknown) => this.host.report(error)); + }); + } + + /** Opens the record's gate and replays queued listener events through the current listeners. */ + openGate(): void { + const gate = this.gate; + this.gate = undefined; + for (const { key, event } of gate?.queue ?? []) this.#deliver(key, event); + } + + // ─── calls ─── + + /** Admission for a call or write carrying `ctx`. */ + admit(ctx: Ctx): void { + if (this.host.failed !== undefined) { + throw new ChordError("unavailable", `${this.host.name} failed and admits no work`, { reason: "host_failed" }); + } + if (!this.host.hasEpoch(ctx.epoch)) throw new ChordError("superseded", `epoch ${ctx.epoch} is retired`); + if (ctx.epoch < this.minEpoch) { + throw new ChordError("upgrade_conflict", `${this.address} was upgraded in epoch ${this.minEpoch}`); + } + if (this.gate !== undefined && !this.gate.grandfathered.has(ctx.rootId)) { + throw new ChordError("upgrade_conflict", `${this.address} is being upgraded`); + } + } + + /** Records that the root of `ctx` touched this record (a reload gate grandfathers it). */ + lease(ctx: Ctx): void { + if (this.leases.has(ctx.rootId)) return; + this.leases.set(ctx.rootId, ctx.epoch); + this.host.touch(ctx.rootId, this); + } + + guard = (ctx: Ctx): void => { + if (this.host.inFactory > 0) throw invalid("read_only", "cells cannot be written while a factory runs"); + if (ctx instanceof CtxImpl) ctx.check(this.host); + if (ctx.signal.aborted) throw new ChordError("aborted", "write with an aborted ctx"); + if (this.status === "disposed") throw new ChordError("gone", `${this.address} is disposed`); + if (ctx.rootId.startsWith("task:")) return; + this.admit(ctx); + }; + + async call(handle: HandleImpl, name: string, args: unknown, ctx: CtxImpl): Promise { + ctx.check(this.host); + if (handle.status === "released" || handle.status === "gone") throw handle.terminalError(); + if (handle.status === "loading") await raceSignal(handle.ready, ctx.signal); + if (handle.status !== "live" || this.status !== "live") { + throw handle.status === "live" + ? new ChordError("gone", `${this.address} is ${this.status}`) + : handle.terminalError(); + } + this.admit(ctx); + const installed = this.behaviors.get(ctx.epoch); + if (installed === undefined) + throw new ChordError("superseded", `${this.address} has no behavior for epoch ${ctx.epoch}`); + const view = handle.viewOf(installed); + const kind = kindOf(view); + const wanted = handle.describe(handle.contract.scope === "remote" ? kind : LOCAL_VARIANT, name); + const fn = view[name]; + if (typeof fn !== "function" || (handle.contract.scope === "remote" && wanted?.kind !== "method")) { + throw new ChordError( + "not_in_variant", + `${this.address}: ${name} is not a method of variant ${kind ?? "(local)"}`, + ); + } + const direct = view === installed.behavior; + const schema = direct ? installed.service.contract.description.variants[kind ?? LOCAL_VARIANT]?.[name] : wanted; + const validated = handle.contract.scope === "remote" && schema?.kind === "method" ? schema : undefined; + // The additive rule assumes a consumer sends only what its own schema knows, and a provider sees only what its + // schema knows: strip with both before validating. + const own = handle.contract.scope === "remote" && wanted?.kind === "method" ? wanted.args : undefined; + const sent = own === undefined ? args : Value.Clean(own, Value.Clone(args)); + const input = validated ? Value.Clean(validated.args, Value.Clone(sent)) : sent; + if (validated) assertValid("args", `${this.address}.${name} args`, validated.args, input); + this.lease(ctx); + const child = ctx.child(this); + let result: unknown; + try { + result = await this.#invoke(() => (fn as (args: unknown, ctx: Ctx) => unknown)(input, child)); + } finally { + child.close(); + } + if (validated) assertValid("result", `${this.address}.${name} result`, validated.result, result); + return result; + } + + /** Runs an invocation (method or listener) that keeps the record alive until it settles. */ + #invoke(fn: () => T | Promise): Promise { + const invocation = (async () => fn())(); + this.#invocations.add(invocation); + const done = (): void => { + this.#invocations.delete(invocation); + if (this.#invocations.size === 0) this.#maybeIdle(); + }; + invocation.then(done, done); + return invocation; + } + + rootEnded(rootId: string): void { + this.leases.delete(rootId); + const gate = this.gate; + if (gate?.grandfathered.delete(rootId) && gate.grandfathered.size === 0) gate.drained(); + } + + // ─── lifetime ─── + + addHold(handle: HandleImpl): void { + this.holds.add(handle); + if (this.#linger !== undefined) clearTimeout(this.#linger); + this.#linger = undefined; + } + + release(handle: HandleImpl): void { + if (this.holds.delete(handle)) this.#maybeIdle(); + } + + /** No holds and no invocations: dispose (after `linger` for a live record). */ + #maybeIdle(): void { + if (this.holds.size > 0 || this.#invocations.size > 0 || this.#linger !== undefined) return; + if (this.status === "activating") { + this.#background(this.dispose(false)); + return; + } + if (this.status !== "live") return; + const linger = this.service?.options.linger ?? 0; + if (linger <= 0) { + this.#background(this.dispose(false)); + return; + } + this.#linger = setTimeout(() => { + this.#linger = undefined; + if (this.holds.size === 0 && this.#invocations.size === 0 && this.status === "live") { + this.#background(this.dispose(false)); + } + }, linger); + this.#linger.unref?.(); + } + + #background(disposal: Promise): void { + disposal.catch((error: unknown) => this.host.report(error)); + } + + /** + * Disposes the record. Forced: handles become gone at once (new calls are rejected) and running invocations are + * cancelled. Either way, resources close only after every invocation settled. + */ + dispose(forced: boolean, reason?: unknown): Promise { + this.#disposal ??= this.#dispose(forced, reason); + return this.#disposal; + } + + async #dispose(forced: boolean, reason: unknown): Promise { + const wasActivating = this.status === "activating"; + this.status = "disposing"; + this.host.forget(this); + if (this.#linger !== undefined) clearTimeout(this.#linger); + this.#linger = undefined; + const error = reason ?? new ChordError("gone", `${this.address} was disposed`); + if (forced) { + for (const handle of this.holds) handle.gone(error); + this.callAbort.abort(error); + } + this.activationAbort.abort(error); + if (wasActivating) await this.ready.catch(noop); + for (const handle of this.holds) handle.gone(error); + const decls = this.allDecls(); + this.stopTasks(); + this.stopListeners(); + await Promise.allSettled([...this.#invocations]); + const registered = [...decls].some((set) => set.registrations.size > 0) && !this.deferredRuntime; + this.behaviors.clear(); + this.decls = new Decls(); + if (registered) this.host.registriesChanged(); + const holds = new Set(); + for (const set of decls) { + for (const derived of set.derives.values()) derived.detach(); + for (const hold of set.holds.values()) holds.add(hold); + } + try { + await runAll( + [ + () => Promise.allSettled(this.#stopped.splice(0)), + ...[...this.openOrder].reverse().map((resource) => () => resource.close(this.openOrder)), + ...[...holds].map((hold) => () => hold.release()), + ], + `${this.address}: disposal failed`, + ); + } finally { + this.status = "disposed"; + } + } +} + +const isReadableLike = (value: unknown): boolean => + value !== null && + typeof value === "object" && + typeof (value as Readable).events === "function" && + typeof (value as Readable).subscribe === "function"; + +/** The remote state-member schemas a behavior exposes, by source. */ +export function memberSchemas(service: Service, behavior: Record): Map, TSchema[]> { + const schemas = new Map, TSchema[]>(); + const contract = service.contract; + if (contract.scope !== "remote") return schemas; + const members = contract.description.variants[kindOf(behavior) ?? ""] ?? {}; + for (const [name, description] of Object.entries(members)) { + const source = behavior[name]; + if (description.kind !== "state" || !isSource(source)) continue; + schemas.set(source, [...(schemas.get(source) ?? []), description.schema]); + } + return schemas; +} + +function validateBehavior(service: Service, behavior: Record): void { + const contract = service.contract; + const fail = (message: string): never => { + throw invalid("behavior", `${contract.name}: ${message}`); + }; + if (behavior === null || typeof behavior !== "object") fail("factory must return an object"); + let members: Readonly>; + if (contract.scope === "remote") { + const kind = behavior.kind; + if (typeof kind !== "string" || contract.description.variants[kind] === undefined) + fail(`unknown variant ${String(kind)}`); + members = contract.description.variants[kind as string]!; + for (const key of Object.keys(behavior)) + if (key !== "kind" && members[key] === undefined) fail(`${key} is not in variant ${String(kind)}`); + } else { + members = contract.description.variants[LOCAL_VARIANT] ?? {}; + } + for (const [name, description] of Object.entries(members)) { + const value = behavior[name]; + if (description.kind === "method" && typeof value !== "function") fail(`${name} must be a method`); + if (description.kind === "state" && !isSource(value) && !isReadableLike(value)) { + fail(`${name} must be a cell, a derivation or a held object's state member`); + } + if (description.kind === "opaque" && value === undefined) fail(`${name} is missing`); + } +} + +// ─── The factory's `self` ──────────────────────────────────────────────────── + +class SelfImpl implements Self { + readonly #record: RecordImpl; + readonly #previous: Decls | undefined; + readonly #decls = new Decls(); + readonly #created: { holds: Hold[]; resources: ResourceImpl[]; derives: Derived[] } = { + holds: [], + resources: [], + derives: [], + }; + + constructor(record: RecordImpl, previous: Decls | undefined) { + this.#record = record; + this.#previous = previous; + } + + run(behavior: unknown): Run { + return { behavior: behavior as Record, decls: this.#decls, created: this.#created }; + } + + /** Iterates named declarations of one kind, rejecting repeated names and wrong descriptors. */ + *#entries( + map: ReadonlyMap, + kind: string, + entries: Readonly>, + descriptor?: string, + ) { + if (entries === null || typeof entries !== "object") + throw new TypeError(`${kind} takes an object of named declarations`); + for (const [name, value] of Object.entries(entries)) { + if (map.has(name)) throw invalid("duplicate", `${this.#record.typeName}: duplicate ${kind} name ${name}`); + if (descriptor !== undefined && declKind(value) !== descriptor) { + throw new TypeError(`${kind}.${name} must be declared with ${descriptor}()`); + } + yield [name, value] as const; + } + } + + state>>>(cells: D): Cells { + const result: Record> = {}; + const host = this.#record.host; + for (const [name, decl] of this.#entries(this.#decls.cells, "state", cells, "cell")) { + const cell = + this.#previous?.cells.get(name) ?? + new Cell(name, decl.schema, decl.initial, this.#record.guard, { + dev: host.dev, + report: (error) => host.report(error), + }); + this.#decls.cells.set(name, cell); + this.#decls.cellSchemas.set(name, decl.schema); + result[name] = cell; + } + return result as Cells; + } + + get(type: C, input: InputOf, options?: GetOptions): Handle { + const host = this.#record.host; + // A prepare rerun (recording mode) binds new holds against the reload's pending contracts: a breaking provider + // change and its updated consumer reload together. + const candidate = this.#previous !== undefined; + const address = host.addressOf(type, input, options?.principal ?? host.principal, options?.at, candidate); + const declared = this.#decls.holds.get(address); + if (declared !== undefined) { + if (declared.contract.fingerprint !== type.fingerprint) { + throw invalid("duplicate", `${this.#record.typeName}: ${type.name} held twice with different contracts`); + } + return declared.proxy as Handle; + } + const existing = this.#previous?.holds.get(address); + if ( + existing !== undefined && + existing.contract.fingerprint === type.fingerprint && + existing.status !== "gone" && + existing.status !== "released" + ) { + this.#decls.holds.set(address, existing); + return existing.proxy as Handle; + } + const handle = host.acquire(type, input, options, "record", this.#record, candidate); + this.#decls.holds.set(address, handle); + this.#created.holds.push(handle); + return handle.proxy as Handle; + } + + resources>>>(resources: D): Resources { + const result: Record = {}; + for (const [name, { spec }] of this.#entries(this.#decls.resources, "resources", resources, "resource")) { + const deps = (spec.deps ?? []).map((dep) => { + const resolved = handleOf(dep) ?? resourceOf(dep); + if (resolved === undefined) { + throw new TypeError(`resource ${name}: deps must be handles or resources of this record`); + } + return resolved; + }); + const existing = this.#previous?.resources.get(name); + const kept = + existing !== undefined && + existing.spec.rev === spec.rev && + existing.exclusive === (spec.exclusive === true) && + existing.deps.length === deps.length && + existing.deps.every((dep, index) => dep === deps[index]); + const resource = kept ? existing : new ResourceImpl(name, spec, deps); + if (!kept) this.#created.resources.push(resource); + this.#decls.resources.set(name, resource); + result[name] = new ResourceFacade(resource, this.#record.host) as unknown as ResourceImpl; + } + return result as unknown as Resources; + } + + listen(listeners: Readonly>): void { + for (const [name, decl] of this.#entries(this.#decls.listeners, "listen", listeners, "listener")) { + this.#decls.listeners.set(name, decl); + } + } + + tasks(tasks: Readonly Promise>>): void { + for (const [name, fn] of this.#entries(this.#decls.tasks, "tasks", tasks)) { + if (typeof fn !== "function") throw new TypeError(`tasks.${name} must be a function`); + this.#decls.tasks.set(name, fn); + } + } + + derive>>>(derivations: D): Derivations { + const result: Record> = {}; + const host = this.#record.host; + for (const [name, { sources, fn }] of this.#entries(this.#decls.derives, "derive", derivations, "derivation")) { + const existing = this.#previous?.derives.get(name); + let derived: Derived; + if ( + existing !== undefined && + existing.sources.length === sources.length && + existing.sources.every((source, index) => source === sources[index]) + ) { + derived = existing; + } else { + derived = new Derived(name, sources, fn, { dev: host.dev, report: (error) => host.report(error) }); + this.#created.derives.push(derived); + } + this.#decls.derives.set(name, derived); + this.#decls.deriveFns.set(name, fn); + result[name] = derived; + } + return result as unknown as Derivations; + } + + expose(resource: Resource): Exposed { + const resolved = resourceOf(resource); + if (resolved === undefined || ![...this.#decls.resources.values()].includes(resolved)) { + throw new TypeError("expose takes a resource declared by this record"); + } + return new ExposedImpl(resolved) as unknown as Exposed; + } + + register(registry: Registry, entries: Readonly>): void { + if (!this.#record.eager) throw invalid("not_eager", `${this.#record.typeName}: only eager records may register`); + const prefixed: Record = {}; + for (const [key, value] of Object.entries(entries)) prefixed[`${registry.name}\u0000${key}`] = value; + for (const [id, value] of this.#entries(this.#decls.registrations, "register", prefixed)) { + this.#decls.registrations.set(id, { + registry: registry as Registry, + key: id.slice(registry.name.length + 1), + value, + }); + } + } +} diff --git a/packages/chord/src/registry.ts b/packages/chord/src/registry.ts new file mode 100644 index 000000000..94fb8c2e0 --- /dev/null +++ b/packages/chord/src/registry.ts @@ -0,0 +1,30 @@ +import type { Ctx } from "./types.ts"; + +export interface Registry { + readonly name: string; + readonly conflict: "error" | "last"; + /** Type carrier only; never set. */ + readonly value?: V; +} + +/** `conflict: "error"` rejects two records registering one key; `"last"` keeps the later registration. */ +export const defineRegistry = (name: string, options: { conflict?: "error" | "last" } = {}): Registry => ({ + name, + conflict: options.conflict ?? "error", +}); + +export type RegistryTable = ReadonlyMap; + +export interface RegistryView { + /** Table of `ctx.epoch`, or of the current epoch without ctx. */ + table(ctx?: Ctx): RegistryTable; + get(key: string, ctx?: Ctx): V | undefined; + /** Called with the current table immediately and whenever a new table is installed. */ + subscribe(listener: (table: RegistryTable, epoch: number) => void): () => void; +} + +export interface Registration { + readonly registry: Registry; + readonly key: string; + readonly value: unknown; +} diff --git a/packages/chord/src/reload.ts b/packages/chord/src/reload.ts new file mode 100644 index 000000000..1aa06555f --- /dev/null +++ b/packages/chord/src/reload.ts @@ -0,0 +1,659 @@ +import type { Cell } from "./cell.ts"; +import { type AnyContract, bind, coverage } from "./contract.ts"; +import { asChordError, ChordError, invalid } from "./errors.ts"; +import type { EpochState, Facet, HostImpl, PluginRef, Serve } from "./host.ts"; +import { holdPublications } from "./publish.ts"; +import { + type Factory, + HandleImpl, + makeService, + memberSchemas, + type RecordImpl, + type ResourceImpl, + type Run, + type ServeOptions, + type Service, + type StagedListeners, +} from "./record.ts"; +import type { RegistryTable } from "./registry.ts"; +import { assertValid, deepEqual } from "./schema.ts"; + +export interface ReloadOptions { + /** Facet names to remove. */ + readonly remove?: readonly string[]; + /** Milliseconds the gate waits for grandfathered roots (default 5000). Missed: the reload aborts. */ + readonly gateDeadline?: number; + /** Milliseconds to wait for the previous epoch to retire first (default 5000). Missed: it is retired anyway. */ + readonly retireDeadline?: number; + /** Server: facet modules for child hosts by host-name prefix (longest wins). Every connected host participates. */ + readonly remote?: Readonly>; + /** Server: facet names to remove on child hosts, by the same host-name prefix rules. */ + readonly remoteRemove?: Readonly>; + /** Server: milliseconds to wait for every child's vote (default: gate + retire deadlines + 5000). */ + readonly voteDeadline?: number; +} + +export interface ReloadReport { + readonly epoch: number; + readonly previous: number; + readonly changed: readonly string[]; + readonly removed: readonly string[]; + /** Addresses of records whose new factory was installed. */ + readonly prepared: readonly string[]; + /** Addresses of records that went through the gate. */ + readonly gated: readonly string[]; + /** + * Cluster reloads: what each participating child host prepared, by host name (its vote; the child commits it after + * the server). Absent for local reloads. + */ + readonly children?: Readonly>; +} + +interface Candidate { + readonly record: RecordImpl; + readonly service: Service; + readonly run: Run; + /** Why the record needs the gate (empty: compatible). */ + readonly reasons: string[]; + /** New exclusive resources and their new dependents, opened at the gate. */ + readonly atGate: ResourceImpl[]; + /** Old exclusive resources closed at the gate. */ + readonly replaced: ResourceImpl[]; + readonly migrated: Map, unknown>; + /** Values of the run's derivations over the candidate (migrated) values, validated. */ + derived: Map; + /** Listener subscriptions made at the end of prepare, activated by the commit. */ + staged: StagedListeners | undefined; +} + +const noop = (): void => {}; + +/** + * Computes the new derivation functions over the candidate values and validates every prospective value (migrated + * cells and derivations) against every schema that will apply after the commit: the cell's new schema and the new + * behavior's remote state-member schemas. A failure fails prepare; the commit then installs without re-checking. + */ +function dryRun(candidate: Candidate): void { + const { run, migrated, service, record } = candidate; + const exposed = memberSchemas(service, run.behavior); + for (const [cell, value] of migrated) { + for (const schema of exposed.get(cell) ?? []) + assertValid("state", `${record.address}: migrated state ${cell.key}`, schema, value); + } + const values = new Map(migrated); + const derives = [...run.decls.derives].sort(([, a], [, b]) => a.seq - b.seq); + for (const [name, derived] of derives) { + const value = derived.compute( + (source) => (values.has(source) ? values.get(source) : source.value), + run.decls.deriveFns.get(name), + ); + for (const schema of exposed.get(derived) ?? []) + assertValid("state", `${record.address}: derived ${name}`, schema, value); + values.set(derived, value); + } + candidate.derived = values; +} + +/** A reload error keeps the underlying code (`contract_mismatch`, `invalid`, ...) and says it aborted the reload. */ +export function reloadError(error: unknown): ChordError { + const cause = asChordError(error, "reload"); + if (/^reload aborted/.test(cause.message.replace(/^[^:]*: /, ""))) return cause; + return new ChordError(cause.code, `reload aborted: ${cause.message}`, { + cause, + reason: cause.reason, + details: cause.details, + }); +} + +/** Serialized: a second reload while one runs is rejected. */ +export async function runReload( + host: HostImpl, + plugins: readonly PluginRef[], + options: ReloadOptions, +): Promise { + const reload = await beginReload(host, plugins, options); + try { + await reload.prepare(); + return reload.commit(); + } finally { + endReload(host); + } +} + +/** Takes the host's reload slot and loads the plugin modules. Pair with `endReload`. */ +export async function beginReload( + host: HostImpl, + plugins: readonly PluginRef[], + options: ReloadOptions, + epoch?: number, +): Promise { + if (host.reloading) { + throw new ChordError("unavailable", "a reload is already running", { reason: "reload_running" }); + } + host.reloading = true; + try { + return new Reload(host, await host.resolve(plugins), options, epoch); + } catch (error) { + host.reloading = false; + throw reloadError(error); + } +} + +export function endReload(host: HostImpl): void { + host.reloading = false; + host.maybeRetire(); +} + +/** One reload on one host: `prepare` (load, prepare, gate), then `commit` or `rollback`. */ +export class Reload { + readonly host: HostImpl; + readonly facets: readonly Facet[]; + readonly options: ReloadOptions; + /** Epoch id to commit (issued by the coordinating server); a local reload allocates one. */ + readonly epoch: number | undefined; + readonly candidates = new Map(); + readonly abort = new AbortController(); + readonly closed: [RecordImpl, ResourceImpl][] = []; + services = new Map(); + changed = new Set(); + removed = new Set(); + tables = new Map>(); + gated: Candidate[] = []; + rolledBack = false; + /** Records of types this reload adds, with their staged listeners: their runtime starts at commit. */ + readonly deferred = new Map(); + + constructor(host: HostImpl, facets: readonly Facet[], options: ReloadOptions, epoch?: number) { + this.host = host; + this.facets = facets; + this.options = options; + this.epoch = epoch; + } + + /** Everything up to the commit. On failure the host is rolled back and the underlying error is thrown. */ + async prepare(): Promise { + try { + await this.waitForRetirement(); + this.load(); + this.checkProviders(); + await this.prepareAll(); + // From here new records of changed types are rejected (upgrade_conflict, retryable) until commit or abort, + // so the candidate set is final once the activations already in flight have been prepared. + this.host.gating = this.changed; + await this.withDeadline(this.prepareAll(), "activations in flight did not finish"); + this.checkHolds(); + this.tables = this.host.buildTables(this.entries()); + await this.gate(); + this.#alive(); + this.#stage(); + } catch (error) { + await this.rollback(); + throw reloadError(error); + } + } + + /** What a successful prepare will commit (a child sends it with its vote). */ + summary(): ReloadReport { + const live = [...this.candidates.values()].filter(({ record }) => record.status === "live"); + return { + epoch: this.host.pending?.id ?? this.host.current.id, + previous: this.host.current.id, + changed: [...this.changed], + removed: [...this.removed], + prepared: live.map(({ record }) => record.address), + gated: this.gated.map(({ record }) => record.address), + }; + } + + /** After every await of prepare: a rollback meanwhile (abort from the coordinator) ends it. */ + #alive(): void { + if (this.rolledBack) throw new ChordError("aborted", "reload aborted while preparing", { reason: "aborted" }); + } + + /** + * Subscribes every new listener now, buffering its events, so a failing source fails prepare instead of leaving a + * half-installed epoch. Records running only the new code (types this reload adds) are staged too. + */ + #stage(): void { + for (const candidate of this.candidates.values()) { + if (candidate.record.status === "live") + candidate.staged = candidate.record.stageListeners(candidate.run.decls); + } + for (const record of this.host.records.values()) { + if (record.status === "live" && record.deferredRuntime) + this.deferred.set(record, record.stageListeners(record.decls)); + } + } + + async waitForRetirement(): Promise { + const host = this.host; + if (host.previous === undefined) return; + if (host.previous.roots.size > 0) { + let timer: ReturnType | undefined; + await new Promise((resolve) => { + host.retired = resolve; + timer = setTimeout(resolve, this.options.retireDeadline ?? 5000); + }); + clearTimeout(timer); + host.retired = noop; + } + // Roots of the dropped epoch still running get `superseded` on their next call. + host.retire(); + } + + /** Builds the candidate service table: unchanged services plus everything the given facets serve. */ + load(): void { + const replaced = new Set([...this.facets.map((facet) => facet.name), ...(this.options.remove ?? [])]); + const current = this.host.current.services; + this.services = new Map([...current].filter(([, service]) => !replaced.has(service.facet))); + for (const facet of this.facets) { + facet.setup( + this.host.facetApi(((type: AnyContract, factory: Factory, options: ServeOptions = {}) => + this.host.serveInto(this.services, makeService(type, factory, options, facet.name))) as Serve), + ); + } + for (const [name, service] of this.services) if (current.get(name) !== service) this.changed.add(name); + for (const name of current.keys()) if (!this.services.has(name)) this.removed.add(name); + this.host.pending = { + id: this.host.nextEpochId(this.epoch), + services: this.services, + tables: new Map(), + roots: new Map(), + }; + } + + async withDeadline(promise: Promise, what: string): Promise { + const deadline = this.options.gateDeadline ?? 5000; + let timer: ReturnType | undefined; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new ChordError("aborted", `${what} within ${deadline} ms`, { reason: "deadline" })), + deadline, + ); + }); + try { + return await Promise.race([promise, timeout]); + } finally { + clearTimeout(timer); + } + } + + /** Same contract version must cover the old description (forgotten bump); facets' `uses` must bind. */ + checkProviders(): void { + for (const name of this.changed) { + const next = this.services.get(name)!.contract; + const old = this.host.current.services.get(name)?.contract; + if (old === undefined || old.version !== next.version) continue; + const reason = coverage(next.description, old.description); + if (reason !== undefined) { + throw new ChordError( + "contract_mismatch", + `${name}@${next.version} changed incompatibly without a version bump (${reason})`, + ); + } + } + const facets = new Map(this.host.facets); + for (const facet of this.facets) facets.set(facet.name, facet); + for (const name of this.options.remove ?? []) facets.delete(name); + for (const facet of facets.values()) { + for (const token of facet.uses ?? []) { + const service = this.services.get(token.name); + if (service !== undefined && this.changed.has(token.name)) + bind(service.contract, service.options.accepts, token); + } + } + } + + /** + * Prepares every live record of a changed type, including records that activate meanwhile, to a fixpoint. Records + * already running the new code (types this reload adds) are skipped. + */ + async prepareAll(): Promise { + for (;;) { + const todo = [...this.host.records.values()].filter( + (record) => + this.changed.has(record.typeName) && + !this.candidates.has(record) && + record.service !== this.services.get(record.typeName) && + (record.status === "live" || record.status === "activating"), + ); + if (todo.length === 0) return; + await Promise.allSettled(todo.map((record) => record.ready)); + this.#alive(); + for (const record of todo) { + if (record.status === "live" && record.service !== this.services.get(record.typeName)) { + await this.prepareRecord(record); + this.#alive(); + } + } + } + } + + /** Runs the new factory in recording mode and opens what can be opened without the gate. */ + async prepareRecord(record: RecordImpl): Promise { + const service = this.services.get(record.typeName)!; + const run = record.run(service, record.decls); + const candidate: Candidate = { + record, + service, + run, + reasons: [], + atGate: [], + replaced: [], + migrated: new Map(), + derived: new Map(), + staged: undefined, + }; + this.candidates.set(record, candidate); + const bumped = service.stateVersion !== record.stateVersion; + if (bumped) candidate.reasons.push(`state version ${record.stateVersion} -> ${service.stateVersion}`); + for (const [name, cell] of run.decls.cells) { + if (record.decls.cells.get(name) !== cell || deepEqual(cell.schema, run.decls.cellSchemas.get(name))) continue; + if (!bumped) { + throw invalid("state", `${record.address}: schema of state ${name} changed without a state-version bump`); + } + } + for (const name of Object.keys(service.options.state?.migrate ?? {})) { + if (!run.decls.cells.has(name)) + throw invalid("state", `${record.address}: migrate for undeclared state ${name}`); + } + const deferred = new Set(); + for (const resource of run.created.resources) { + if (resource.exclusive || resource.deps.some((dep) => deferred.has(dep as ResourceImpl))) + deferred.add(resource); + } + for (const resource of deferred) { + candidate.atGate.push(resource); + const old = record.decls.resources.get(resource.key); + if (resource.exclusive && old?.exclusive && old.state === "open") candidate.replaced.push(old); + } + if (deferred.size > 0) candidate.reasons.push("exclusive resource replaced"); + if (candidate.reasons.length === 0) dryRun(candidate); + await record.open( + run.created.resources.filter((resource) => !deferred.has(resource)), + run.created.holds, + this.abort.signal, + ); + } + + /** + * Every hold that survives into the new epoch must bind to the new provider. Browser holds don't block a reload: + * after the commit they get `upgrade_required` instead. + */ + checkHolds(): void { + const dropped = new Set(); + for (const { record, run } of this.candidates.values()) { + for (const hold of record.decls.holds.values()) + if (![...run.decls.holds.values()].includes(hold)) dropped.add(hold); + } + for (const record of this.host.records.values()) { + const service = this.services.get(record.typeName); + if (service === undefined || !this.changed.has(record.typeName)) continue; + for (const hold of record.holds) { + // The host's bootstrap hold follows this provider at commit; it is not an old-code consumer. + if (hold === this.host.eager.get(record.typeName)) continue; + if (!dropped.has(hold) && hold.owner !== "client") + bind(service.contract, service.options.accepts, hold.contract); + } + } + } + + entries(): [RecordImpl, Run["decls"]][] { + return this.host + .liveDecls(true) + .filter(([record]) => !this.removed.has(record.typeName)) + .map(([record, decls]) => [record, this.candidates.get(record)?.run.decls ?? decls]); + } + + /** + * Admission cut, wait for grandfathered roots (deadline), replace exclusive resources, migrate into private + * copies. Old tasks of gated records are stopped at the cut; listener events are queued until commit or abort. + */ + async gate(): Promise { + this.gated = [...this.candidates.values()].filter( + (candidate) => candidate.reasons.length > 0 && candidate.record.status === "live", + ); + const drained = this.gated.map(({ record }) => { + const grandfathered = new Set(record.leases.keys()); + return new Promise((resolve) => { + record.gate = { grandfathered, queue: [], drained: resolve }; + record.stopTasks(); + if (grandfathered.size === 0) resolve(); + }); + }); + const deadline = this.options.gateDeadline ?? 5000; + let timer: ReturnType | undefined; + const timedOut = await Promise.race([ + Promise.all(drained).then(() => false), + new Promise((resolve) => { + timer = setTimeout(() => resolve(true), deadline); + }), + ]); + clearTimeout(timer); + if (timedOut) { + const blocking = this.gated.flatMap(({ record }) => [...(record.gate?.grandfathered ?? [])]); + throw new ChordError( + "aborted", + `gate deadline (${deadline} ms) missed; blocking roots: ${blocking.join(", ")}`, + { + reason: "gate_deadline", + details: { blocking }, + }, + ); + } + this.#alive(); + for (const candidate of this.gated) { + const { record } = candidate; + for (const old of candidate.replaced) { + // Owned by the rollback before closing: a failing close must still be restored. + this.closed.push([record, old]); + await old.close(record.openOrder); + } + await record.open(candidate.atGate, [], this.abort.signal); + this.#alive(); + } + for (const candidate of this.gated) this.migrate(candidate); + } + + migrate(candidate: Candidate): void { + const { record, service, run } = candidate; + const migrate = service.options.state?.migrate ?? {}; + const bumped = service.stateVersion !== record.stateVersion; + for (const [name, cell] of run.decls.cells) { + if (record.decls.cells.get(name) !== cell) continue; + const schema = run.decls.cellSchemas.get(name)!; + const fn = migrate[name]; + let value: unknown; + if (bumped && fn !== undefined) value = fn(structuredClone(cell.value), record.stateVersion); + else if (!deepEqual(cell.schema, schema)) value = cell.value; + else continue; + assertValid("state", `${record.address}: migrated state ${name}`, schema, value); + candidate.migrated.set(cell, value); + } + dryRun(candidate); + } + + async rollback(): Promise { + if (this.rolledBack) return; + this.rolledBack = true; + this.abort.abort(new ChordError("aborted", "reload aborted")); + const report = (error: unknown): void => this.host.report(error); + for (const candidate of this.candidates.values()) candidate.staged?.discard(); + for (const staged of this.deferred.values()) staged.discard(); + for (const { record, run } of this.candidates.values()) { + record.pending.delete(run.decls); + await record.unwind(run).catch(report); + } + // Replaced exclusive resources are restored; one that cannot be is marked unavailable, never silently closed. + for (const [record, resource] of this.closed) { + if (resource.state !== "closed") continue; + await resource.open(new AbortController().signal, record.openOrder).catch((error: unknown) => { + resource.markUnavailable(); + report( + new ChordError("unavailable", `${record.address}: resource ${resource.key} could not be restored`, { + cause: error, + }), + ); + }); + } + for (const { record } of this.gated) { + if (record.gate === undefined) continue; + record.openGate(); + for (const name of record.decls.tasks.keys()) record.startTask(name, this.host.current.id); + } + // Records of types this reload would have added lose their code. + const reason = new ChordError("gone", "reload aborted; type not served"); + for (const record of [...this.host.records.values()]) { + if (!this.host.current.services.has(record.typeName)) await record.dispose(true, reason).catch(report); + } + this.host.pending = undefined; + this.host.gating = undefined; + } + + /** + * Publishes the new epoch in one synchronous step. Everything fallible ran in prepare (factories, opens, migration + * and derivation checks against the prospective schemas, listener subscriptions), so this only installs. Observers + * (state, shapes, registries, listener roots) are notified after the whole step. If an install still throws, the + * epoch is half-published: the host fails terminally and admits no more work. + */ + commit(beforeStart?: () => void): ReloadReport { + try { + return holdPublications(() => { + const report = this.#publish(); + // Installed, not yet started: the coordinator queues its commit frames here, so nothing this epoch starts + // (tasks, eager activations, then the held listener/registry/state notifications) can reach a child first. + beforeStart?.(); + for (const start of this.#starts.splice(0)) start(); + return report; + }); + } catch (error) { + this.host.fail(error); + throw reloadError(error); + } + } + + /** Work that may start new-epoch roots, run after installation (and after the coordinator's commit frames). */ + readonly #starts: (() => void)[] = []; + + #publish(): ReloadReport { + const host = this.host; + const previous = host.current; + const next: EpochState = host.pending!; + host.pending = undefined; + host.epochs.set(next.id, next); + host.previous = previous; + host.current = next; + const prepared: string[] = []; + for (const record of [...host.records.values()]) { + if (this.removed.has(record.typeName)) continue; + const candidate = this.candidates.get(record); + if (candidate === undefined) { + const installed = record.behaviors.get(previous.id); + if (installed !== undefined) record.behaviors.set(next.id, { ...installed, epoch: next.id }); + continue; + } + if (record.status !== "live") continue; + this.install(candidate, next.id); + prepared.push(record.address); + } + for (const { record, run, staged } of this.candidates.values()) { + if (record.status === "live") continue; + staged?.discard(); + record.pending.delete(run.decls); + record.unwind(run).catch((error: unknown) => host.report(error)); + } + // Records of types this reload adds start their runtime now. + for (const [record, staged] of this.deferred) { + if (record.status !== "live") { + staged.discard(); + continue; + } + record.deferredRuntime = false; + staged.activate(); + this.#starts.push(() => record.startTasks()); + } + host.installTables(next, this.tables, true); + const reason = new ChordError("gone", "type no longer served"); + for (const record of [...host.records.values()]) { + if (this.removed.has(record.typeName)) + record.dispose(true, reason).catch((error: unknown) => host.report(error)); + } + for (const name of this.removed) host.eager.delete(name); + for (const facet of this.facets) host.facets.set(facet.name, facet); + for (const name of this.options.remove ?? []) host.facets.delete(name); + host.gating = undefined; + for (const [name, service] of this.services) { + if (service.options.eager && this.changed.has(name) && host.eager.get(name)?.record?.status !== "live") { + this.#starts.push(() => host.startEager(service)); + } + } + return { + epoch: next.id, + previous: previous.id, + changed: [...this.changed], + removed: [...this.removed], + prepared, + gated: this.gated.map(({ record }) => record.address), + }; + } + + install(candidate: Candidate, epoch: number): void { + const { record, run, service } = candidate; + const gated = candidate.reasons.length > 0; + record.pending.delete(run.decls); + record.stopTasks(); + if (gated) { + // Old derivations would compute over migrated values; the old behavior can no longer run anyway. + const kept = new Set(run.decls.derives.values()); + for (const derived of record.decls.derives.values()) if (!kept.has(derived)) derived.detach(); + } + record.decls = run.decls; + record.service = service; + record.stateVersion = service.stateVersion; + record.behaviors.set(epoch, { epoch, service, behavior: run.behavior, decls: run.decls }); + if (gated) { + // Old code can never run over migrated state: drop the previous behavior now, not at retirement. + record.minEpoch = epoch; + for (const old of [...record.behaviors.keys()]) if (old !== epoch) this.host.retireBehavior(record, old); + } + // Schemas first, then the values validated against them in prepare. + record.refreshSchemas(); + for (const [name, derived] of run.decls.derives) derived.setFn(run.decls.deriveFns.get(name)!); + for (const [name, cell] of run.decls.cells) { + const schema = run.decls.cellSchemas.get(name)!; + if (candidate.migrated.has(cell)) cell.reset(candidate.migrated.get(cell), schema); + else cell.schema = schema; + } + const created = new Set(run.created.derives); + for (const derived of run.created.derives) derived.attach(candidate.derived); + for (const derived of run.decls.derives.values()) if (!created.has(derived)) derived.refresh(); + candidate.staged?.activate(); + this.#starts.push(() => record.startTasks()); + record.openGate(); + const eager = this.host.eager.get(record.typeName); + if (eager?.record === record && eager.contract.fingerprint !== service.contract.fingerprint) { + const replacement = new HandleImpl(record, service.contract, "host"); + // Acquire first: releasing the only hold must not dispose/reopen the surviving eager record. + record.addHold(replacement); + this.host.eager.set(record.typeName, replacement); + eager.release(); + } + for (const handle of [...record.holds]) { + // Browser holds whose contract no longer binds are told to reload their code (they never block a reload). + if (handle.owner === "client") { + try { + bind(service.contract, service.options.accepts, handle.contract); + } catch (error) { + handle.gone( + new ChordError("upgrade_required", `${record.typeName} changed; reload the web facets`, { + cause: error, + }), + ); + record.release(handle); + continue; + } + } + handle.shapeChanged(); + } + } +} diff --git a/packages/chord/src/remote.ts b/packages/chord/src/remote.ts new file mode 100644 index 000000000..042447a3a --- /dev/null +++ b/packages/chord/src/remote.ts @@ -0,0 +1,357 @@ +import { Value } from "typebox/value"; +import type { AnyContract } from "./contract.ts"; +import { applyImmutable, type Decoder, decoder, type WireOp } from "./delta/index.ts"; +import { ChordError } from "./errors.ts"; +import { + CtxImpl, + HANDLE_BASE, + type Hold, + type HoldOwner, + holdsByProxy, + notifyStatus, + raceSignal, + type StatusListener, +} from "./record.ts"; +import type { Ctx, HandleStatus, Principal, Readable, Shape, StateEvent } from "./types.ts"; +import { fromWireError, type RootRef, type WireError } from "./wire.ts"; + +const noop = (): void => {}; + +export interface StateSink { + snap?: unknown; + reset?: unknown; + ops?: readonly WireOp[]; + error?: WireError; +} + +/** What a remote handle needs from the link it is bound to (implemented by Connection). */ +export interface RemoteChannel { + hold(handle: RemoteHandle): number; + release(id: number): void; + call(hold: number, member: string, args: unknown, root: RootRef | undefined, signal: AbortSignal): Promise; + sub(hold: number, member: string, sink: (frame: StateSink) => void): number; + unsub(id: number): void; +} + +/** What a remote handle needs from its host. */ +export interface RemoteHost { + readonly inFactory: number; + report(error: unknown): void; + root(fn: (ctx: Ctx) => T | Promise, options?: { signal?: AbortSignal }): Promise; + forgetRemote(handle: RemoteHandle): void; +} + +/** + * A state member of a remote handle: hydrated while subscribed (snapshot, then decoded ops); `undefined` before. + * Re-subscribing after a reconnect delivers the new snapshot as a reset. + */ +class RemoteReadable implements Readable { + readonly #handle: RemoteHandle; + readonly #name: string; + readonly #values = new Set<(value: unknown) => void>(); + readonly #events = new Set<(event: StateEvent) => void>(); + #sub: { readonly channel: RemoteChannel; readonly id: number } | undefined; + #decoder: Decoder = decoder(); + #value: unknown; + + constructor(handle: RemoteHandle, name: string) { + this.#handle = handle; + this.#name = name; + } + + get value(): unknown { + return this.#value; + } + + subscribe(listener: (value: unknown) => void): () => void { + this.#values.add(listener); + this.#safe(() => listener(this.#value)); + this.bind(); + return () => this.#remove(this.#values, listener); + } + + events(listener: (event: StateEvent) => void): () => void { + this.#events.add(listener); + this.#safe(() => listener({ type: "snap", value: this.#value })); + this.bind(); + return () => this.#remove(this.#events, listener); + } + + /** Subscribes upstream while someone listens and the handle is bound; drops a subscription on an old link. */ + bind(): void { + const listening = this.#values.size + this.#events.size > 0; + // Nobody listens: the value would go stale, so forget it. The next subscriber sees `undefined`, then the snapshot. + if (!listening) this.#value = undefined; + const binding = listening ? this.#handle.binding : undefined; + if (binding !== undefined && this.#sub?.channel === binding.channel) return; + if (this.#sub !== undefined) { + if (this.#sub.channel !== binding?.channel || binding === undefined) this.#sub.channel.unsub(this.#sub.id); + this.#sub = undefined; + } + if (binding === undefined) return; + const channel = binding.channel; + this.#decoder = decoder(); + const id = channel.sub(binding.id, this.#name, (frame) => this.#receive(frame)); + this.#sub = { channel, id }; + } + + /** The link is gone: keep the last value, resubscribe on the next binding. */ + detach(): void { + this.#sub = undefined; + } + + stop(): void { + this.#values.clear(); + this.#events.clear(); + this.bind(); + } + + #receive(frame: StateSink): void { + if (frame.error !== undefined) { + this.#handle.host.report(fromWireError(frame.error)); + return; + } + let event: StateEvent; + if (frame.ops === undefined && !("snap" in frame) && !("reset" in frame)) return; + if (frame.ops !== undefined) { + const ops = this.#decoder.decode(frame.ops); + this.#value = applyImmutable(this.#value, ops); + event = { type: "ops", ops, value: this.#value }; + } else { + // Listeners already saw a snapshot (`undefined` or an older value), so every upstream snapshot is a reset. + this.#decoder = decoder(); + this.#value = "snap" in frame ? frame.snap : frame.reset; + event = { type: "reset", value: this.#value }; + } + for (const listener of [...this.#events]) this.#safe(() => listener(event)); + for (const listener of [...this.#values]) this.#safe(() => listener(this.#value)); + } + + #remove(set: Set, listener: T): void { + set.delete(listener); + this.bind(); + } + + #safe(fn: () => void): void { + try { + fn(); + } catch (error) { + this.#handle.host.report(error); + } + } +} + +/** A hold on an object on another host, bound to a link. Late-bound like a local handle. */ +export class RemoteHandle implements Hold { + readonly record = undefined; + readonly host: RemoteHost; + readonly contract: AnyContract; + readonly at: string; + readonly input: unknown; + /** Forwarded principal (omitted for the host's own principal). */ + readonly principal: Principal | undefined; + readonly owner: HoldOwner; + readonly address: string; + readonly proxy: object; + readonly ready: Promise; + status: HandleStatus = "loading"; + error: unknown; + shape: Shape | undefined; + binding: { readonly channel: RemoteChannel; readonly id: number } | undefined; + readonly shapeListeners = new Set<(shape: Shape) => void>(); + readonly #readables = new Map(); + readonly #callers = new Map Promise>(); + #settleReady: { resolve: () => void; reject: (error: unknown) => void } | undefined; + #live: { promise: Promise; resolve: () => void; reject: (error: unknown) => void }; + + constructor( + host: RemoteHost, + fields: { + contract: AnyContract; + at: string; + input: unknown; + principal: Principal | undefined; + owner: HoldOwner; + address: string; + }, + ) { + this.host = host; + this.contract = fields.contract; + this.at = fields.at; + this.input = fields.input; + this.principal = fields.principal; + this.owner = fields.owner; + this.address = fields.address; + this.ready = new Promise((resolve, reject) => { + this.#settleReady = { resolve, reject }; + }); + this.ready.catch(noop); + this.#live = this.#newLive(); + this.proxy = new Proxy(this, { + get: (target, property) => { + if (HANDLE_BASE.has(property)) { + const value: unknown = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + } + if (typeof property !== "string" || property === "then") return undefined; + return target.member(property); + }, + }); + holdsByProxy.set(this.proxy, this); + } + + get kind(): string | undefined { + return this.shape?.kind; + } + + onShape(listener: (shape: Shape) => void): () => void { + this.shapeListeners.add(listener); + return () => this.shapeListeners.delete(listener); + } + + onStatus(listener: StatusListener): () => void { + this.#statusListeners.add(listener); + return () => this.#statusListeners.delete(listener); + } + + readonly #statusListeners = new Set(); + + #statusChanged(): void { + notifyStatus(this.#statusListeners, this.status, this.error, (error) => this.host.report(error)); + } + + release(): void { + if (this.status === "released" || this.status === "gone") return; + this.status = "released"; + this.#statusChanged(); + const error = new ChordError("released", `handle ${this.address} released`); + this.#settleReady?.reject(error); + this.#live.reject(error); + for (const readable of this.#readables.values()) readable.stop(); + if (this.binding !== undefined) this.binding.channel.release(this.binding.id); + this.binding = undefined; + this.host.forgetRemote(this); + } + + // ─── driven by the link ─── + + bound(channel: RemoteChannel, id: number): void { + this.binding = { channel, id }; + } + + shaped(shape: Shape): void { + if (this.status === "released" || this.status === "gone") return; + this.shape = shape; + const wasLoading = this.status === "loading"; + this.status = "live"; + if (wasLoading) this.#statusChanged(); + if (wasLoading) { + this.#settleReady?.resolve(); + this.#settleReady = undefined; + this.#live.resolve(); + } + for (const readable of this.#readables.values()) readable.bind(); + for (const listener of [...this.shapeListeners]) { + try { + listener(shape); + } catch (error) { + this.host.report(error); + } + } + } + + gone(error: unknown): void { + if (this.status === "released" || this.status === "gone") return; + this.status = "gone"; + this.error = error; + this.#statusChanged(); + this.binding = undefined; + this.#settleReady?.reject(error); + this.#live.reject(error); + for (const readable of this.#readables.values()) readable.stop(); + this.host.forgetRemote(this); + } + + /** The link went down but may come back (upstream reconnect): wait, then re-hold. */ + detached(): void { + if (this.status === "released" || this.status === "gone") return; + this.binding = undefined; + for (const readable of this.#readables.values()) readable.detach(); + if (this.status === "live") { + this.status = "loading"; + this.#live = this.#newLive(); + this.#statusChanged(); + } + } + + // ─── members ─── + + member(name: string): unknown { + const description = Object.values(this.contract.description.variants) + .map((members) => members[name]) + .find((member) => member !== undefined); + if (description?.kind === "method") return this.#caller(name); + if (description?.kind === "state") { + let readable = this.#readables.get(name); + if (readable === undefined) { + readable = new RemoteReadable(this, name); + this.#readables.set(name, readable); + } + return readable; + } + return undefined; + } + + #caller(name: string): (args: unknown, ctx?: Ctx) => Promise { + let caller = this.#callers.get(name); + if (caller === undefined) { + caller = (args, ctx) => { + if (this.host.inFactory > 0) + throw new ChordError("not_ready", `${this.address}.${name} called inside a factory`); + if (ctx instanceof CtxImpl) { + try { + ctx.check(ctx.host); + } catch (error) { + return Promise.reject(error); + } + } + if (ctx === undefined || (ctx instanceof CtxImpl && ctx.holder)) { + return this.host.root((root) => this.#call(name, args, root), { signal: ctx?.signal }); + } + return this.#call(name, args, ctx); + }; + this.#callers.set(name, caller); + } + return caller; + } + + async #call(name: string, args: unknown, ctx: Ctx): Promise { + for (;;) { + if (this.status === "released" || this.status === "gone") { + throw this.status === "gone" ? this.error : new ChordError("released", `handle ${this.address} released`); + } + if (this.status === "loading") await raceSignal(this.#live.promise, ctx.signal); + const binding = this.binding; + if (binding === undefined) continue; + const kind = this.shape?.kind; + if (kind === undefined || this.contract.description.variants[kind]?.[name]?.kind !== "method") { + throw new ChordError("not_in_variant", `${this.address}: ${name} is not a method of variant ${kind}`); + } + // Send only what the consumer's own schema knows (the additive rule's premise; see RecordImpl.call). + const description = this.contract.description.variants[kind]?.[name]; + const sent = description?.kind === "method" ? Value.Clean(description.args, Value.Clone(args)) : args; + return binding.channel.call(binding.id, name, sent, { rootId: ctx.rootId, epoch: ctx.epoch }, ctx.signal); + } + } + + #newLive(): { promise: Promise; resolve: () => void; reject: (error: unknown) => void } { + let resolve = noop; + let reject: (error: unknown) => void = noop; + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); + promise.catch(noop); + return { promise, resolve, reject }; + } +} diff --git a/packages/chord/src/schema.ts b/packages/chord/src/schema.ts new file mode 100644 index 000000000..7883969e5 --- /dev/null +++ b/packages/chord/src/schema.ts @@ -0,0 +1,219 @@ +import type { TSchema } from "typebox"; +import { Value } from "typebox/value"; +import { type ChordErrorReason, invalid } from "./errors.ts"; + +/** JSON Schema as data. TypeBox schemas are plain JSON Schema objects. */ +type Schema = { readonly [key: string]: unknown }; + +const asSchema = (schema: unknown): Schema => + schema !== null && typeof schema === "object" ? (schema as Schema) : schema === false ? { not: {} } : {}; + +/** Canonical JSON: sorted object keys, `undefined` members dropped. */ +export function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`; + const keys = Object.keys(value) + .filter((key) => (value as Record)[key] !== undefined) + .sort(); + return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson((value as Record)[key])}`).join(",")}}`; +} + +/** Non-cryptographic 64-bit hash (two FNV-1a lanes). Contracts are trusted, so collisions are not an attack surface. */ +export function hash64(text: string): string { + let a = 0x811c9dc5; + let b = 0x01000193 ^ 0x5bd1e995; + for (let index = 0; index < text.length; index++) { + const code = text.charCodeAt(index); + a = Math.imul(a ^ code, 0x01000193); + b = Math.imul(b ^ code, 0x5bd1e995) ^ (b >>> 13); + } + return (a >>> 0).toString(16).padStart(8, "0") + (b >>> 0).toString(16).padStart(8, "0"); +} + +export function deepEqual(a: unknown, b: unknown): boolean { + return a === b || canonicalJson(a) === canonicalJson(b); +} + +/** Throws `invalid` (with `reason`) and the first schema errors when `value` does not match. */ +export function assertValid(reason: ChordErrorReason, what: string, schema: TSchema, value: unknown): void { + if (Value.Check(schema, value)) return; + const errors = [...Value.Errors(schema, value)].slice(0, 3); + const summary = errors.map((error) => `${error.instancePath || "/"} ${error.message}`).join("; "); + throw invalid(reason, `${what}: ${summary}`, errors); +} + +/** Applies defaults, strips unknown fields, validates. Returns a fresh value. */ +export function normalize(schema: TSchema, value: unknown, what: string): unknown { + const normalized = Value.Clean(schema, Value.Default(schema, Value.Clone(value ?? {}))); + assertValid("input", what, schema, normalized); + return normalized; +} + +/** Canonical identity of a normalized value: fields equal to their schema default are omitted. */ +export function canonicalIdentity(schema: TSchema, value: unknown): string { + return canonicalJson(stripDefaults(asSchema(schema), value)); +} + +function stripDefaults(schema: Schema, value: unknown): unknown { + if (Array.isArray(value)) { + const items = schema.items; + if (Array.isArray(items)) return value.map((item, index) => stripDefaults(asSchema(items[index]), item)); + return value.map((item) => stripDefaults(asSchema(items), item)); + } + if (value === null || typeof value !== "object") return value; + const properties = asSchema(schema.properties); + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + const property = asSchema(properties[key]); + if ("default" in property && deepEqual(property.default, item)) continue; + result[key] = stripDefaults(property, item); + } + return result; +} + +const CONSTRAINTS = [ + "minLength", + "maxLength", + "pattern", + "format", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "multipleOf", + "minItems", + "maxItems", + "uniqueItems", + "minProperties", + "maxProperties", +] as const; + +const typesOf = (schema: Schema): string[] | undefined => + typeof schema.type === "string" ? [schema.type] : Array.isArray(schema.type) ? (schema.type as string[]) : undefined; + +/** Keywords `fits` reasons about. Any other validation keyword in the provider's schema forces exact equality. */ +const KNOWN: ReadonlySet = new Set([ + ...CONSTRAINTS, + "type", + "properties", + "required", + "additionalProperties", + "patternProperties", + "items", + "additionalItems", + "const", + "enum", + "anyOf", + "parameters", + "returnType", +]); + +/** Annotations: no effect on validity. */ +const ANNOTATIONS: ReadonlySet = new Set([ + "title", + "description", + "default", + "examples", + "$id", + "$schema", + "$comment", + "readOnly", + "writeOnly", + "deprecated", +]); + +const withoutAnnotations = (schema: Schema): Schema => + Object.fromEntries(Object.entries(schema).filter(([key]) => !ANNOTATIONS.has(key))); + +/** + * Conservative structural subtyping: true only if every value valid under `sub` is valid under `sup`, where object + * fields missing from `sub` but optional in `sup` count as fine (unknown fields are stripped at the provider, see + * `RecordImpl.call`). A provider schema using a keyword this function does not model (`not`, `allOf`, `contains`, + * `$ref`, ...) or an `additionalProperties` schema only fits an identical consumer schema. + */ +export function fits(subSchema: unknown, supSchema: unknown): boolean { + const sub = withoutAnnotations(asSchema(subSchema)); + const sup = withoutAnnotations(asSchema(supSchema)); + if (Object.keys(sup).length === 0) return true; + if (deepEqual(sub, sup)) return true; + if (Object.keys(sup).some((key) => !KNOWN.has(key))) return false; + if (sup.additionalProperties !== undefined && typeof sup.additionalProperties !== "boolean") return false; + if (Array.isArray(sub.anyOf)) return sub.anyOf.every((branch) => fits(branch, sup)); + if (Array.isArray(sup.anyOf)) return sup.anyOf.some((branch) => fits(sub, branch)); + if ("const" in sup) return "const" in sub && deepEqual(sub.const, sup.const); + if (Array.isArray(sup.enum)) { + const values = "const" in sub ? [sub.const] : Array.isArray(sub.enum) ? sub.enum : undefined; + return values?.every((item) => (sup.enum as unknown[]).some((allowed) => deepEqual(allowed, item))) ?? false; + } + for (const key of CONSTRAINTS) if (key in sup && !deepEqual(sub[key], sup[key])) return false; + const supTypes = typesOf(sup); + const subTypes = typesOf(sub); + if (supTypes === undefined || subTypes === undefined) return false; + if (!subTypes.every((type) => supTypes.includes(type) || (type === "integer" && supTypes.includes("number")))) { + return false; + } + if ("const" in sub || Array.isArray(sub.enum)) return true; + if (subTypes.includes("object") && !objectFits(sub, sup)) return false; + if (subTypes.includes("array") && !arrayFits(sub, sup)) return false; + if (subTypes.includes("function")) return functionFits(sub, sup); + return true; +} + +function objectFits(sub: Schema, sup: Schema): boolean { + const subProperties = asSchema(sub.properties); + const supProperties = asSchema(sup.properties); + const subRequired = new Set((sub.required as string[] | undefined) ?? []); + for (const key of (sup.required as string[] | undefined) ?? []) if (!subRequired.has(key)) return false; + for (const [key, property] of Object.entries(supProperties)) { + if (key in subProperties) { + if (!fits(subProperties[key], property)) return false; + } else if (typeof sub.additionalProperties === "object" && !fits(sub.additionalProperties, property)) { + // The consumer may send this name with any value its `additionalProperties` allows. Open consumer objects + // are fine: unknown names are stripped with the consumer's own schema before the call (`RecordImpl.call`). + return false; + } + } + if (sup.additionalProperties === false) { + if (sub.additionalProperties !== false) return false; + if (Object.keys(subProperties).some((key) => !(key in supProperties))) return false; + } + const supPatterns = asSchema(sup.patternProperties); + if (Object.keys(supPatterns).length > 0) { + // Every value the consumer may put under a matching name must fit, and the consumer must not allow arbitrary + // extra names (it would then allow any value under a matching one). + const subPatterns = asSchema(sub.patternProperties); + if (sub.additionalProperties !== false && Object.keys(subPatterns).length === 0) return false; + for (const [pattern, property] of Object.entries(supPatterns)) { + const matcher = new RegExp(pattern, "u"); + for (const [key, subProperty] of Object.entries(subProperties)) { + if (!(key in supProperties) && matcher.test(key) && !fits(subProperty, property)) return false; + } + if ( + Object.keys(subPatterns).length > 0 && + (!(pattern in subPatterns) || !fits(subPatterns[pattern], property)) + ) { + return false; + } + } + for (const pattern of Object.keys(subPatterns)) if (!(pattern in supPatterns)) return false; + } + return true; +} + +function arrayFits(sub: Schema, sup: Schema): boolean { + if (Array.isArray(sup.items)) { + if (!Array.isArray(sub.items) || sub.items.length !== sup.items.length) return false; + return sub.items.every((item, index) => fits(item, (sup.items as unknown[])[index])); + } + if (Array.isArray(sub.items)) return sub.items.every((item) => fits(item, sup.items)); + return fits(sub.items, sup.items); +} + +function functionFits(sub: Schema, sup: Schema): boolean { + const subParameters = (sub.parameters as unknown[] | undefined) ?? []; + const supParameters = (sup.parameters as unknown[] | undefined) ?? []; + // A function accepting fewer parameters is fine; each shared parameter is contravariant. + if (subParameters.length > supParameters.length) return false; + if (!subParameters.every((parameter, index) => fits(supParameters[index], parameter))) return false; + return fits(sub.returnType, sup.returnType); +} diff --git a/packages/chord/src/services/consumer.ts b/packages/chord/src/services/consumer.ts deleted file mode 100644 index 1b6b2f121..000000000 --- a/packages/chord/src/services/consumer.ts +++ /dev/null @@ -1,713 +0,0 @@ -import { awaitWithContext, BACKGROUND_CONTEXT } from "../context/index.ts"; -import type { Op } from "../delta/index.ts"; -import type { - Context, - JsonValue, - RemoteServiceBinding, - RemoteServiceBindingOptions, - RemoteServiceTransport, - ReplicatedState, - Service, - ServiceInstanceAddress, - ServiceInstanceSnapshot, - ServiceMemberSnapshot, - ServiceProviderUpdate, - ServiceSubscription, - ServiceSubscriptionSnapshot, -} from "../types.ts"; -import { RemoteServiceError } from "./errors.ts"; -import { ServiceSlot } from "./handle.ts"; -import { InstanceDirectory, type InstanceDirectoryEntry } from "./instances.ts"; -import { ReplicatedStateReplica, serviceDeliveryContext } from "./state.ts"; - -type ErrorReporter = (error: Error) => void; -type ServiceMemberKind = ServiceMemberSnapshot["kind"]; - -class MemberSlot { - readonly #serviceId: string; - readonly #member: string; - readonly #invoke: (args: readonly JsonValue[], context: Context) => Promise; - readonly #state: ReplicatedStateReplica; - readonly #isActive: () => boolean; - readonly #assertAccess: () => void; - readonly value: unknown; - #kind: ServiceMemberKind | undefined; - #expectedKind: ServiceMemberKind | undefined; - - constructor( - serviceId: string, - member: string, - invoke: (args: readonly JsonValue[], context: Context) => Promise, - isActive: () => boolean, - assertAccess: () => void, - reportError: ErrorReporter, - ) { - this.#serviceId = serviceId; - this.#member = member; - this.#invoke = invoke; - this.#isActive = isActive; - this.#assertAccess = assertAccess; - this.#state = new ReplicatedStateReplica(reportError); - const callable = (): void => {}; - this.value = new Proxy(callable, { - apply: (_target, _thisArg, args) => this.#call(args), - get: (_target, property) => { - if (property === "value") { - this.#assertAccess(); - this.#expect("state"); - return this.#state.value; - } - if (property === "subscribe") return this.#subscribe.bind(this); - if (property === Symbol.toStringTag) return "RemoteServiceMember"; - if (property === "then") return undefined; - return undefined; - }, - }); - } - - setDescription(kind: ServiceMemberKind): void { - if (this.#kind !== undefined && this.#kind !== kind) { - throw new Error(`Remote service member ${this.#serviceId}.${this.#member} changed kind`); - } - this.#kind = kind; - if (this.#expectedKind !== undefined && this.#expectedKind !== kind) { - throw new RemoteServiceError( - "service_member_mismatch", - `Remote service member ${this.#serviceId}.${this.#member} is ${kind}, not ${this.#expectedKind}`, - ); - } - } - - hydrate(sequence: number, ops: readonly Op[], context: Context): void { - this.setDescription("state"); - this.#state.hydrate(sequence, ops, context); - } - - update(sequence: number, ops: readonly Op[], context: Context): void { - this.setDescription("state"); - this.#state.update(sequence, ops, context); - } - - clear(): void { - this.#state.clear(); - } - - #subscribe(listener: Parameters["subscribe"]>[0]): () => void { - this.#assertAccess(); - if (typeof listener !== "function") { - throw new TypeError("Replicated state subscription listener must be a function"); - } - this.#expect("state"); - return this.#state.subscribe(listener); - } - - #expect(kind: ServiceMemberKind): void { - if (this.#expectedKind !== undefined && this.#expectedKind !== kind) { - throw new RemoteServiceError( - "service_member_mismatch", - `Remote service member ${this.#serviceId}.${this.#member} was used as two different kinds`, - ); - } - this.#expectedKind = kind; - if (this.#kind !== undefined && this.#kind !== kind) { - throw new RemoteServiceError( - "service_member_mismatch", - `Remote service member ${this.#serviceId}.${this.#member} is ${this.#kind}, not ${kind}`, - ); - } - } - - #call(args: unknown[]): Promise { - this.#assertAccess(); - this.#expect("method"); - if (!this.#isActive()) { - return Promise.reject( - new RemoteServiceError("service_stale_instance", `Remote service ${this.#serviceId} binding is closed`), - ); - } - const context = args.at(-1); - if (!isContext(context)) { - return Promise.reject( - new RemoteServiceError( - "service_invalid_value", - `Remote service method ${this.#serviceId}.${this.#member} requires a trailing Context`, - ), - ); - } - const businessArgs = args.slice(0, -1) as JsonValue[]; - return this.#invoke(businessArgs, context); - } -} - -class ServiceFacade { - readonly #serviceId: string; - readonly #address: ServiceInstanceAddress | undefined; - readonly #transport: RemoteServiceTransport; - readonly #reportError: ErrorReporter; - readonly #slots = new Map(); - readonly #descriptions = new Map(); - readonly #isActive: () => boolean; - readonly #assertAccess: () => void; - readonly proxy: object; - - constructor( - serviceId: string, - address: ServiceInstanceAddress | undefined, - transport: RemoteServiceTransport, - isActive: () => boolean, - assertAccess: () => void, - reportError: ErrorReporter, - ) { - this.#serviceId = serviceId; - this.#address = address; - this.#transport = transport; - this.#isActive = isActive; - this.#assertAccess = assertAccess; - this.#reportError = reportError; - this.proxy = new Proxy(Object.create(null) as object, { - get: (_target, property) => { - if (typeof property !== "string") return undefined; - return this.#slot(property).value; - }, - }); - } - - install(snapshot: ServiceInstanceSnapshot, context: Context): void { - if (!sameAddress(snapshot.instance, this.#address)) - throw new Error("Remote service snapshot has the wrong address"); - const members = validateMembers(snapshot.members); - for (const name of this.#slots.keys()) { - if (!members.has(name)) { - throw new RemoteServiceError( - "service_member_not_found", - `Unknown remote service member ${this.#serviceId}.${name}`, - ); - } - } - this.#descriptions.clear(); - for (const [name, member] of members) this.#descriptions.set(name, member.kind); - for (const member of members.values()) { - const slot = this.#slots.get(member.name); - if (member.kind === "state") { - (slot ?? this.#slot(member.name)).hydrate(member.sequence, member.ops, context); - } else { - slot?.setDescription(member.kind); - } - } - } - - update(member: string, sequence: number, ops: readonly Op[], context: Context): void { - if (this.#descriptions.get(member) !== "state") { - throw new Error(`Remote service update targets non-state member ${this.#serviceId}.${member}`); - } - this.#slot(member).update(sequence, ops, context); - } - - clear(): void { - for (const slot of this.#slots.values()) slot.clear(); - } - - #slot(member: string): MemberSlot { - let slot = this.#slots.get(member); - if (slot !== undefined) return slot; - slot = new MemberSlot( - this.#serviceId, - member, - (args, context) => - this.#transport.invoke( - { - serviceId: this.#serviceId, - ...(this.#address === undefined ? {} : { instance: this.#address }), - member, - args, - }, - context, - ), - this.#isActive, - this.#assertAccess, - this.#reportError, - ); - const kind = this.#descriptions.get(member); - if (kind !== undefined) slot.setDescription(kind); - this.#slots.set(member, slot); - return slot; - } -} - -interface SingletonBinding { - facade: ServiceFacade; - subscription?: ServiceSubscription; - starting?: Promise; - active: boolean; - revision: number; -} - -interface KeyedInstance extends InstanceDirectoryEntry { - readonly facade: ServiceFacade; -} - -class KeyedBinding { - readonly #service: Service; - readonly #transport: RemoteServiceTransport; - readonly #reportError: ErrorReporter; - readonly #assertAccess: () => void; - readonly #onEmpty: () => void; - readonly #instances: InstanceDirectory; - #subscription: ServiceSubscription | undefined; - #starting: Promise | undefined; - #closed = false; - #bound: boolean; - #revision = 0; - - constructor( - service: Service, - transport: RemoteServiceTransport, - reportError: ErrorReporter, - assertAccess: () => void, - onEmpty: () => void, - bound: boolean, - ) { - this.#service = service; - this.#transport = transport; - this.#reportError = reportError; - this.#assertAccess = assertAccess; - this.#onEmpty = onEmpty; - this.#bound = bound; - this.#instances = new InstanceDirectory({ ready: false, onError: reportError }); - } - - observe(handler: (service: T, context: Context) => void | Promise): () => void { - if (this.#closed) throw new Error("Remote keyed service binding is closed"); - let stopped = false; - const stop = this.#instances.observe((service, context) => { - const slot = new ServiceSlot(this.#service.id, true); - void slot.bind(service as object); - return handler( - slot.view(() => { - this.#assertAccess(); - if (stopped || context.abortSignal?.aborted) { - throw new RemoteServiceError( - "service_stale_instance", - `Remote service ${this.#service.id} observation is closed`, - ); - } - }), - context, - ); - }); - if (this.#bound && this.#starting === undefined) { - const revision = this.#revision; - const starting = this.#start(revision); - this.#starting = starting; - void starting.catch((error: unknown) => { - if (!this.#closed && this.#revision === revision && this.#bound) this.#reportError(toError(error)); - }); - } - return () => { - if (stopped) return; - stopped = true; - stop(); - if (this.#instances.observerCount === 0) this.#onEmpty(); - }; - } - - async rebind(bound: boolean, context: Context): Promise { - if (this.#closed) return; - this.#bound = bound; - const revision = ++this.#revision; - await this.#reset(context, false); - if (this.#closed || this.#revision !== revision || this.#bound !== bound) return; - if (bound && this.#instances.observerCount > 0) { - const starting = this.#start(revision); - this.#starting = starting; - await starting; - } - } - - ready(): Promise { - return this.#starting ?? Promise.resolve(); - } - - async close(context: Context): Promise { - if (this.#closed) return; - this.#closed = true; - this.#revision += 1; - await this.#reset(context, true); - this.#instances.dispose(); - } - - async #reset(context: Context, waitForStarting: boolean): Promise { - this.#instances.reset(); - const starting = this.#starting; - this.#starting = undefined; - const subscription = this.#subscription; - this.#subscription = undefined; - await Promise.all([ - waitForStarting ? starting?.catch(() => {}) : undefined, - subscription === undefined ? undefined : subscription.close(context), - ]); - } - - async #start(revision: number): Promise { - const subscription = await this.#transport.subscribe( - this.#service.id, - "keyed", - (update, context) => { - if (this.#revision === revision) this.#update(update, context); - }, - BACKGROUND_CONTEXT, - ); - if (this.#closed || !this.#bound || this.#revision !== revision) { - await subscription.close(BACKGROUND_CONTEXT); - return; - } - this.#subscription = subscription; - if (subscription.snapshot.mode !== "keyed" || subscription.snapshot.serviceId !== this.#service.id) { - throw new Error(`Remote service ${this.#service.id} returned the wrong keyed snapshot`); - } - for (const snapshot of subscription.snapshot.instances) this.#spawn(snapshot, serviceDeliveryContext()); - subscription.activate(); - this.#instances.ready(); - } - - #update(update: ServiceProviderUpdate, context: Context): void { - if (this.#closed) return; - try { - switch (update.type) { - case "reset": { - validateResetSnapshot(update.snapshot, this.#service.id, "keyed"); - const snapshots = new Map( - update.snapshot.instances.map((snapshot) => [snapshot.instance!.key, snapshot]), - ); - for (const instance of [...this.#instances.values()]) { - if (snapshots.get(instance.key)?.instance?.generation !== instance.generation) { - this.#instances.remove(instance); - } - } - for (const snapshot of snapshots.values()) { - const instance = this.#instances.get(snapshot.instance!.key); - if (instance === undefined) this.#spawn(snapshot, context); - else instance.facade.install(snapshot, context); - } - break; - } - case "unavailable": - case "replaced": - throw new Error("Keyed service received a singleton lifecycle update"); - case "spawned": - this.#spawn(update.instance, context); - break; - case "closed": { - const instance = this.#instances.get(update.instance.key); - if (instance?.generation === update.instance.generation) this.#instances.remove(instance); - break; - } - case "state": { - if (update.instance === undefined) throw new Error("Keyed state update has no instance address"); - const instance = this.#instances.get(update.instance.key); - if (instance?.generation !== update.instance.generation) return; - instance.facade.update(update.member, update.sequence, update.ops, context); - break; - } - } - } catch (error) { - this.#reportError(toError(error)); - } - } - - #spawn(snapshot: ServiceInstanceSnapshot, context: Context): void { - const address = snapshot.instance; - if (address === undefined) throw new Error("Keyed service instance snapshot has no address"); - let active = true; - const facade = new ServiceFacade( - this.#service.id, - address, - this.#transport, - () => active && !this.#closed, - this.#assertAccess, - this.#reportError, - ); - facade.install(snapshot, context); - this.#instances.replace({ - key: address.key, - generation: address.generation, - service: facade.proxy, - facade, - deactivate: () => { - active = false; - facade.clear(); - }, - }); - } -} - -export class RemoteServiceBindingImpl implements RemoteServiceBinding { - readonly #transport: RemoteServiceTransport; - readonly #allowlist = new Set(); - readonly #reportError: ErrorReporter; - readonly #modes = new Map(); - readonly #assertAccess: () => void; - readonly #singletons = new Map(); - readonly #keyed = new Map>(); - #bound: boolean; - #readinessRevision = 0; - #bindingTransition = Promise.resolve(); - #disposed = false; - - constructor(options: RemoteServiceBindingOptions) { - this.#transport = options.transport; - const ids = options.services.map(({ id }) => id); - if (new Set(ids).size !== ids.length) throw new TypeError("Remote service binding has duplicate service IDs"); - for (const id of ids) this.#allowlist.add(id); - this.#reportError = options.onError ?? (() => {}); - this.#assertAccess = options.assertAccess ?? (() => {}); - this.#bound = options.bound ?? true; - } - - use(service: Service): T { - this.#assertRemotable(service); - this.#assertAvailable(service.id, "singleton"); - let binding = this.#singletons.get(service.id); - if (binding !== undefined) return binding.facade.proxy as T; - binding = { facade: undefined as unknown as ServiceFacade, active: true, revision: 0 }; - binding.facade = new ServiceFacade( - service.id, - undefined, - this.#transport, - () => binding!.active && !this.#disposed && this.#bound, - () => this.#assertHandleAccess(), - this.#reportError, - ); - this.#singletons.set(service.id, binding); - this.#readinessRevision += 1; - if (this.#bound) { - const revision = binding.revision; - const starting = this.#startSingleton(service.id, binding, revision); - binding.starting = starting; - void starting.catch((error: unknown) => { - if (binding.active && binding.revision === revision && !this.#disposed && this.#bound) { - this.#reportError(toError(error)); - } - }); - } - return binding.facade.proxy as T; - } - - observe(service: Service, handler: (service: T, context: Context) => void | Promise): () => void { - this.#assertRemotable(service); - this.#assertAvailable(service.id, "keyed"); - let binding = this.#keyed.get(service.id) as KeyedBinding | undefined; - if (binding === undefined) { - binding = new KeyedBinding( - service, - this.#transport, - this.#reportError, - () => this.#assertHandleAccess(), - () => { - if (this.#keyed.get(service.id) !== binding) return; - this.#keyed.delete(service.id); - this.#readinessRevision += 1; - void binding!.close(BACKGROUND_CONTEXT).catch(this.#reportError); - }, - this.#bound, - ); - this.#keyed.set(service.id, binding as KeyedBinding); - this.#readinessRevision += 1; - } - return binding.observe(handler); - } - - async ready(context: Context): Promise { - if (this.#disposed) throw new Error("Remote service binding is disposed"); - while (true) { - const revision = this.#readinessRevision; - const starts = [ - this.#bindingTransition, - ...[...this.#singletons.values()].flatMap((binding) => - binding.starting === undefined ? [] : [binding.starting], - ), - ...[...this.#keyed.values()].map((binding) => binding.ready()), - ]; - await awaitWithContext( - Promise.all(starts).then(() => undefined), - context, - ); - if (this.#disposed) throw new Error("Remote service binding is disposed"); - if (revision === this.#readinessRevision) return; - } - } - - async rebind(bound: boolean, context: Context): Promise { - if (this.#disposed) throw new Error("Remote service binding is disposed"); - this.#bound = bound; - this.#readinessRevision += 1; - const transitions: Promise[] = []; - for (const [serviceId, binding] of this.#singletons) { - binding.revision += 1; - binding.facade.clear(); - const subscription = binding.subscription; - delete binding.subscription; - const revision = binding.revision; - const starting = (async () => { - await subscription?.close(context); - if (bound) await this.#startSingleton(serviceId, binding, revision); - })(); - binding.starting = starting; - transitions.push(starting); - } - for (const binding of this.#keyed.values()) transitions.push(binding.rebind(bound, context)); - const completion = (async () => { - const results = await Promise.allSettled(transitions); - const errors = results.filter((result): result is PromiseRejectedResult => result.status === "rejected"); - if (errors.length > 0) - throw new AggregateError( - errors.map(({ reason }) => reason), - "Failed to rebind services", - ); - })(); - this.#bindingTransition = completion; - await completion; - } - - async dispose(context: Context): Promise { - if (this.#disposed) return; - this.#disposed = true; - const closes: Promise[] = []; - for (const binding of this.#singletons.values()) { - binding.active = false; - binding.facade.clear(); - if (binding.starting) closes.push(binding.starting.catch(() => {})); - if (binding.subscription) closes.push(Promise.resolve(binding.subscription.close(context))); - } - for (const binding of this.#keyed.values()) closes.push(binding.close(context)); - this.#singletons.clear(); - this.#keyed.clear(); - const results = await Promise.allSettled(closes); - const errors = results.filter((result): result is PromiseRejectedResult => result.status === "rejected"); - if (errors.length > 0) - throw new AggregateError( - errors.map(({ reason }) => reason), - "Failed to dispose services", - ); - } - - async #startSingleton(serviceId: string, binding: SingletonBinding, revision: number): Promise { - const subscription = await this.#transport.subscribe( - serviceId, - "singleton", - (update, context) => { - if (!binding.active || binding.revision !== revision) return; - try { - if (update.type === "reset") { - validateResetSnapshot(update.snapshot, serviceId, "singleton"); - const snapshot = update.snapshot.instances[0]; - if (snapshot === undefined) binding.facade.clear(); - else binding.facade.install(snapshot, context); - } else if (update.type === "unavailable") { - binding.facade.clear(); - } else if (update.type === "replaced") { - if (update.snapshot.instance !== undefined) { - throw new Error("Singleton replacement has an instance address"); - } - binding.facade.install(update.snapshot, context); - } else if (update.type === "state" && update.instance === undefined) { - binding.facade.update(update.member, update.sequence, update.ops, context); - } - } catch (error) { - this.#reportError(toError(error)); - } - }, - BACKGROUND_CONTEXT, - ); - if (!binding.active || this.#disposed || !this.#bound || binding.revision !== revision) { - await subscription.close(BACKGROUND_CONTEXT); - return; - } - binding.subscription = subscription; - const snapshot = subscription.snapshot; - if (snapshot.mode !== "singleton" || snapshot.serviceId !== serviceId || snapshot.instances.length !== 1) { - throw new Error(`Remote service ${serviceId} returned an invalid singleton snapshot`); - } - binding.facade.install(snapshot.instances[0]!, serviceDeliveryContext()); - subscription.activate(); - } - - #assertHandleAccess(): void { - if (this.#disposed) throw new Error("Remote service binding is disposed"); - this.#assertAccess(); - } - - #assertRemotable(service: { readonly id: string; readonly local: boolean }): void { - if (service.local) throw new RemoteServiceError("service_not_allowed", `Service ${service.id} is process-local`); - } - - #assertAvailable(serviceId: string, mode: "singleton" | "keyed"): void { - if (this.#disposed) throw new Error("Remote service binding is disposed"); - if (!this.#allowlist.has(serviceId)) { - throw new RemoteServiceError("service_not_allowed", `Remote service ${serviceId} is not allowlisted`); - } - const existing = this.#modes.get(serviceId); - if (existing !== undefined && existing !== mode) { - throw new RemoteServiceError( - "service_mode_mismatch", - `Remote service ${serviceId} is already used as ${existing}`, - ); - } - this.#modes.set(serviceId, mode); - } -} - -function validateResetSnapshot( - snapshot: ServiceSubscriptionSnapshot, - serviceId: string, - mode: "singleton" | "keyed", -): void { - if ( - snapshot.serviceId !== serviceId || - snapshot.mode !== mode || - (mode === "singleton" && snapshot.instances.length > 1) - ) { - throw new Error("Remote service reset has the wrong service or mode"); - } - const keys = new Set(); - for (const instance of snapshot.instances) { - if (mode === "singleton" ? instance.instance !== undefined : instance.instance === undefined) { - throw new Error("Remote service reset has an invalid instance address"); - } - if (instance.instance !== undefined) { - if (keys.has(instance.instance.key)) throw new Error("Remote service reset repeats an instance key"); - keys.add(instance.instance.key); - } - for (const member of instance.members) { - if (member.kind === "state" && (member.ops.length !== 1 || member.ops[0]?.[0] !== "r")) { - throw new Error("Remote service reset must contain full root replacements"); - } - } - } -} - -function validateMembers(members: readonly ServiceMemberSnapshot[]): ReadonlyMap { - const result = new Map(); - for (const member of members) { - if (member.name.length === 0 || result.has(member.name)) { - throw new Error("Remote service has invalid member descriptions"); - } - result.set(member.name, member); - } - return result; -} - -function sameAddress(left: ServiceInstanceAddress | undefined, right: ServiceInstanceAddress | undefined): boolean { - if (left === undefined || right === undefined) return left === right; - return left.key === right.key && left.generation === right.generation; -} - -function isContext(value: unknown): value is Context { - if (typeof value !== "object" || value === null) return false; - const candidate = value as Partial; - return typeof candidate.value === "function" && typeof candidate.toString === "function"; -} - -function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); -} diff --git a/packages/chord/src/services/errors.ts b/packages/chord/src/services/errors.ts deleted file mode 100644 index 81134943a..000000000 --- a/packages/chord/src/services/errors.ts +++ /dev/null @@ -1,26 +0,0 @@ -export const REMOTE_SERVICE_ERROR_CODES = [ - "service_not_allowed", - "service_not_found", - "service_mode_mismatch", - "service_member_not_found", - "service_member_mismatch", - "service_instance_not_found", - "service_stale_instance", - "service_invalid_value", -] as const; - -export type RemoteServiceErrorCode = (typeof REMOTE_SERVICE_ERROR_CODES)[number]; - -export function isRemoteServiceErrorCode(value: unknown): value is RemoteServiceErrorCode { - return typeof value === "string" && REMOTE_SERVICE_ERROR_CODES.includes(value as RemoteServiceErrorCode); -} - -export class RemoteServiceError extends Error { - readonly code: RemoteServiceErrorCode; - - constructor(code: RemoteServiceErrorCode, message: string) { - super(message); - this.name = "RemoteServiceError"; - this.code = code; - } -} diff --git a/packages/chord/src/services/handle.ts b/packages/chord/src/services/handle.ts deleted file mode 100644 index 129cc887c..000000000 --- a/packages/chord/src/services/handle.ts +++ /dev/null @@ -1,113 +0,0 @@ -interface ResolvedValue { - readonly value: unknown; - readonly receiver: object; -} - -type ValueResolver = () => ResolvedValue; - -/** Host-owned mutable target with consumer-owned guarded views. */ -export class ServiceSlot { - readonly #serviceId: string; - readonly #wrapObjects: boolean; - #implementation: object | undefined; - - constructor(serviceId: string, wrapObjects: boolean) { - this.#serviceId = serviceId; - this.#wrapObjects = wrapObjects; - } - - view(assertAccess: () => void): T { - return new ServiceView(this, assertAccess, this.#wrapObjects).proxy as T; - } - - bind(implementation: object): void { - this.#implementation = implementation; - } - - unbind(): void { - this.#implementation = undefined; - } - - resolve(property: PropertyKey, assertAccess: () => void): ResolvedValue { - assertAccess(); - const implementation = this.#implementation; - if (implementation === undefined) throw new Error(`Service ${this.#serviceId} is disconnected`); - return { - value: Reflect.get(implementation, property, implementation), - receiver: implementation, - }; - } -} - -class ServiceView { - readonly #slot: ServiceSlot; - readonly #assertAccess: () => void; - readonly #wrapObjects: boolean; - readonly #members = new Map(); - readonly proxy: object; - - constructor(slot: ServiceSlot, assertAccess: () => void, wrapObjects: boolean) { - this.#slot = slot; - this.#assertAccess = assertAccess; - this.#wrapObjects = wrapObjects; - this.proxy = new Proxy(Object.create(null) as object, { - get: (_target, property) => this.#getMember(property), - }); - } - - #getMember(property: PropertyKey): unknown { - const resolve = (): ResolvedValue => this.#slot.resolve(property, this.#assertAccess); - const current = resolve().value; - if (!isObject(current) || (typeof current !== "function" && !this.#wrapObjects)) return current; - let member = this.#members.get(property); - if (member === undefined) { - member = new ValueView(resolve, typeof current === "function"); - this.#members.set(property, member); - } - return member.proxy; - } -} - -class ValueView { - readonly #resolve: ValueResolver; - readonly #children = new Map(); - readonly proxy: object; - - constructor(resolve: ValueResolver, callable: boolean) { - this.#resolve = resolve; - const target: object = callable ? () => undefined : Object.create(null); - this.proxy = new Proxy(target, { - apply: (_target, _thisArg, args) => this.#invoke(args), - get: (_target, property) => this.#get(property), - }); - } - - #invoke(args: unknown[]): unknown { - const { value, receiver } = this.#resolve(); - if (typeof value !== "function") throw new TypeError("Service member is not callable"); - return Reflect.apply(value, receiver, args); - } - - #get(property: PropertyKey): unknown { - const resolve = (): ResolvedValue => { - const parent = this.#resolve(); - if (!isObject(parent.value)) throw new TypeError("Service member does not have properties"); - return { - value: Reflect.get(parent.value, property, parent.value), - receiver: parent.value, - }; - }; - const current = resolve().value; - if (typeof current !== "function") return current; - let child = this.#children.get(property); - if (child === undefined) { - child = new ValueView(resolve, true); - this.#children.set(property, child); - } - return child.proxy; - } -} - -function isObject(value: unknown): value is object { - return (typeof value === "object" && value !== null) || typeof value === "function"; -} diff --git a/packages/chord/src/services/instances.ts b/packages/chord/src/services/instances.ts deleted file mode 100644 index 1b7fbf01d..000000000 --- a/packages/chord/src/services/instances.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { BACKGROUND_CONTEXT, withCancel } from "../context/index.ts"; -import type { Context } from "../types.ts"; - -export interface InstanceDirectoryEntry { - readonly key: string; - readonly generation: number; - readonly service: object; - deactivate(): void; -} - -interface Observer { - readonly handler: (service: object, context: Context) => void | Promise; - readonly tasks: Map void }>; - closed: boolean; -} - -/** Owns keyed instance lifetime and the cancellable tasks observing those instances. */ -export class InstanceDirectory { - readonly #entries = new Map(); - readonly #observers = new Set(); - readonly #reportError: (error: Error) => void; - #ready: boolean; - #disposed = false; - - constructor(options: { readonly ready: boolean; readonly onError: (error: Error) => void }) { - this.#ready = options.ready; - this.#reportError = options.onError; - } - - get observerCount(): number { - return this.#observers.size; - } - - values(): IterableIterator { - return this.#entries.values(); - } - - get(key: string): TEntry | undefined { - return this.#entries.get(key); - } - - insert(entry: TEntry): void { - this.#assertActive(); - if (this.#entries.has(entry.key)) { - throw new Error(`Keyed service already has a live instance with key ${entry.key}`); - } - this.#entries.set(entry.key, entry); - if (this.#ready) this.#startAll(entry); - } - - replace(entry: TEntry): void { - this.#assertActive(); - const previous = this.#entries.get(entry.key); - if (previous !== undefined) { - if (previous.generation === entry.generation) { - throw new Error("Keyed service repeated a live generation"); - } - this.#remove(previous); - } - this.#entries.set(entry.key, entry); - if (this.#ready) this.#startAll(entry); - } - - remove(entry: TEntry): void { - if (this.#entries.get(entry.key) !== entry) return; - this.#remove(entry); - } - - ready(): void { - this.#assertActive(); - if (this.#ready) return; - this.#ready = true; - for (const entry of this.#entries.values()) this.#startAll(entry); - } - - reset(): void { - if (this.#disposed) return; - this.#ready = false; - for (const entry of [...this.#entries.values()]) this.#remove(entry); - } - - observe(handler: (service: T, context: Context) => void | Promise): () => void { - this.#assertActive(); - const observer: Observer = { - handler: handler as Observer["handler"], - tasks: new Map(), - closed: false, - }; - this.#observers.add(observer); - if (this.#ready) { - for (const entry of this.#entries.values()) this.#start(observer, entry); - } - return () => { - if (observer.closed) return; - observer.closed = true; - for (const task of observer.tasks.values()) task.cancel(); - observer.tasks.clear(); - this.#observers.delete(observer); - }; - } - - dispose(): void { - if (this.#disposed) return; - this.#disposed = true; - for (const observer of this.#observers) { - observer.closed = true; - for (const task of observer.tasks.values()) task.cancel(); - observer.tasks.clear(); - } - this.#observers.clear(); - for (const entry of [...this.#entries.values()]) { - entry.deactivate(); - } - this.#entries.clear(); - } - - #remove(entry: TEntry): void { - if (this.#entries.get(entry.key) !== entry) return; - this.#entries.delete(entry.key); - entry.deactivate(); - for (const observer of this.#observers) { - observer.tasks.get(entry)?.cancel(); - observer.tasks.delete(entry); - } - } - - #startAll(entry: TEntry): void { - for (const observer of this.#observers) this.#start(observer, entry); - } - - #start(observer: Observer, entry: TEntry): void { - if (observer.closed || observer.tasks.has(entry)) return; - const { context, cancel } = withCancel(BACKGROUND_CONTEXT); - observer.tasks.set(entry, { cancel }); - try { - void Promise.resolve(observer.handler(entry.service, context)).catch((error: unknown) => { - if (!context.abortSignal?.aborted) this.#reportError(toError(error)); - }); - } catch (error) { - if (!context.abortSignal?.aborted) this.#reportError(toError(error)); - } - } - - #assertActive(): void { - if (this.#disposed) throw new Error("Keyed service directory is disposed"); - } -} - -function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); -} diff --git a/packages/chord/src/services/loopback.ts b/packages/chord/src/services/loopback.ts deleted file mode 100644 index 4d4d623b0..000000000 --- a/packages/chord/src/services/loopback.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { RemoteServiceTransport } from "../types.ts"; -import type { RemoteServiceProvider } from "./provider.ts"; - -/** Connects a provider to a binding without changing remote service semantics. */ -export function createLoopbackServiceTransport(provider: RemoteServiceProvider): RemoteServiceTransport { - return { - invoke: (call, context) => provider.invoke(call, context), - subscribe: async (serviceId, mode, listener) => { - const subscription = provider.subscribe(serviceId, mode, listener); - return { - snapshot: subscription.snapshot, - activate: () => subscription.activate(), - close: () => subscription.close(), - }; - }, - }; -} diff --git a/packages/chord/src/services/provider.ts b/packages/chord/src/services/provider.ts deleted file mode 100644 index 936900611..000000000 --- a/packages/chord/src/services/provider.ts +++ /dev/null @@ -1,633 +0,0 @@ -import type { - Context, - JsonValue, - RemoteServiceContract, - Service, - ServiceCall, - ServiceCatalogueEntry, - ServiceInstanceAddress, - ServiceInstanceSnapshot, - ServiceMemberSnapshot, - ServiceMode, - ServiceProviderUpdate, - ServiceSubscription, - ServiceSubscriptionSnapshot, -} from "../types.ts"; -import { RemoteServiceError } from "./errors.ts"; -import { serviceDeliveryContext } from "./state.ts"; -import { getReplicatedStateInternals, type ReplicatedStateInternals } from "./state-internals.ts"; -import { decodeServiceControlCall } from "./wire.ts"; - -type RemoteMethod = (...args: unknown[]) => unknown; - -type InstanceMember = - | { readonly kind: "method"; readonly method: RemoteMethod } - | { readonly kind: "state"; readonly state: ReplicatedStateInternals }; - -type ServiceMemberKind = InstanceMember["kind"]; - -interface ClassifiedRemoteServiceImplementation { - readonly implementation: object; - readonly members: Map; -} - -interface ProviderInstance { - readonly address?: ServiceInstanceAddress; - readonly implementation: object; - readonly members: ReadonlyMap; - readonly removeMemberListeners: readonly (() => void)[]; - active: boolean; -} - -interface ProviderSubscriber { - readonly listener: (update: ServiceProviderUpdate, context: Context) => void; - readonly buffer: { readonly update: ServiceProviderUpdate; readonly context: Context }[]; - readonly snapshotSequences: Map; - active: boolean; - draining: boolean; - terminated: boolean; - closed: boolean; -} - -interface ServiceProviderDefinition { - readonly service: { readonly id: string; readonly local?: boolean }; - readonly mode: ServiceMode; -} - -interface ServiceRegistration { - readonly serviceId: string; - readonly mode: ServiceMode; - singleton?: ProviderInstance; - singletonShape?: ReadonlyMap; - readonly instances: Map; - readonly generations: Map; - readonly subscribers: Set; -} - -export type ServiceUpdatePublisher = ( - subscriptionId: string, - update: ServiceProviderUpdate, - context: Context, -) => void | Promise; - -/** Hosts one provider for one remote consumer and owns that consumer's subscriptions. */ -export interface RemoteServiceEndpoint { - invoke(call: ServiceCall, publish: ServiceUpdatePublisher, context: Context): Promise; - dispose(): void; -} - -export class RemoteServiceProvider { - readonly #catalogue: readonly ServiceCatalogueEntry[]; - readonly #registrations = new Map(); - #disposed = false; - - constructor(entries: readonly (ServiceProviderDefinition | { readonly id: string })[]) { - const definitions = entries.map( - (entry): ServiceProviderDefinition => ("service" in entry ? entry : { service: entry, mode: "singleton" }), - ); - for (const { service } of definitions) { - if ("local" in service && service.local === true) { - throw new TypeError(`Local service ${service.id} cannot be published remotely`); - } - } - const ids = definitions.map(({ service }) => service.id); - if (new Set(ids).size !== ids.length) throw new TypeError("Remote service catalogue contains duplicate IDs"); - this.#catalogue = Object.freeze( - definitions.map(({ service, mode }) => Object.freeze({ serviceId: service.id, mode })), - ); - for (const { service, mode } of definitions) { - this.#registrations.set(service.id, { - serviceId: service.id, - mode, - instances: new Map(), - generations: new Map(), - subscribers: new Set(), - }); - } - } - - get catalogue(): readonly ServiceCatalogueEntry[] { - return this.#catalogue; - } - - provide(service: Service, implementation: NoInfer>): void { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - const registration = this.#registration(service.id, "singleton"); - if (registration.singleton !== undefined) { - throw new RemoteServiceError("service_mode_mismatch", `Remote service ${service.id} already has a provider`); - } - const classified = classifyRemoteServiceImplementation(registration.serviceId, implementation); - const shape = serviceMemberShape(classified.members); - this.#assertSingletonShape(registration, shape); - registration.singleton = this.#createInstance(registration, classified, undefined); - registration.singletonShape = shape; - } - - /** Disconnect one singleton while preserving active subscriptions and remote facades. */ - withdraw(service: Service): void { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - const registration = this.#registration(service.id, "singleton"); - const previous = registration.singleton; - if (previous === undefined) return; - previous.active = false; - for (const remove of previous.removeMemberListeners) remove(); - delete registration.singleton; - this.#emit(registration, { type: "unavailable" }); - } - - /** Check a singleton replacement without changing the active provider. */ - validateReplacement(service: Service, implementation: NoInfer>): void { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - const registration = this.#registration(service.id, "singleton"); - const classified = classifyRemoteServiceImplementation(registration.serviceId, implementation); - this.#assertSingletonShape(registration, serviceMemberShape(classified.members)); - } - - /** Replace one singleton without making its stable remote facade unavailable. */ - replace(service: Service, implementation: NoInfer>): void { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - const registration = this.#registration(service.id, "singleton"); - const classified = classifyRemoteServiceImplementation(registration.serviceId, implementation); - const shape = serviceMemberShape(classified.members); - this.#assertSingletonShape(registration, shape); - const replacement = this.#createInstance(registration, classified, undefined); - const previous = registration.singleton; - if (previous !== undefined) { - previous.active = false; - for (const remove of previous.removeMemberListeners) remove(); - } - registration.singleton = replacement; - registration.singletonShape = shape; - this.#emit(registration, { type: "replaced", snapshot: this.#snapshotInstance(replacement) }); - } - - use(service: Service): T { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - const registration = this.#registrations.get(service.id); - if (registration?.mode !== "singleton" || registration.singleton === undefined) { - throw new RemoteServiceError("service_not_found", `Remote service ${service.id} has no local provider`); - } - return registration.singleton.implementation as T; - } - - spawn(service: Service, key: string, implementation: NoInfer>): () => void { - this.#assertActive(); - this.#assertRemotable(service); - this.#assertAllowed(service.id); - if (key.length === 0) throw new TypeError("Remote service instance key must not be empty"); - const registration = this.#registration(service.id, "keyed"); - if (registration.instances.has(key)) { - throw new RemoteServiceError( - "service_mode_mismatch", - `Remote service ${service.id} already has a live instance with key ${key}`, - ); - } - const generation = (registration.generations.get(key) ?? 0) + 1; - registration.generations.set(key, generation); - const address = { key, generation } satisfies ServiceInstanceAddress; - const classified = classifyRemoteServiceImplementation(registration.serviceId, implementation); - const instance = this.#createInstance(registration, classified, address); - registration.instances.set(key, instance); - this.#emit(registration, { type: "spawned", instance: this.#snapshotInstance(instance) }); - let closed = false; - return () => { - if (closed) return; - closed = true; - if (registration.instances.get(key) !== instance) return; - instance.active = false; - for (const remove of instance.removeMemberListeners) remove(); - registration.instances.delete(key); - this.#emit(registration, { type: "closed", instance: address }); - }; - } - - async invoke(call: ServiceCall, context: Context): Promise { - this.#assertActive(); - this.#assertAllowed(call.serviceId); - const registration = this.#registrations.get(call.serviceId); - if (registration === undefined) { - throw new RemoteServiceError("service_not_found", `Unknown remote service ${call.serviceId}`); - } - const instance = this.#resolveInstance(registration, call.instance); - const member = instance.members.get(call.member); - if (member === undefined) { - throw new RemoteServiceError( - "service_member_not_found", - `Unknown remote service member ${call.serviceId}.${call.member}`, - ); - } - if (member.kind !== "method") { - throw new RemoteServiceError( - "service_member_mismatch", - `Remote service member ${call.serviceId}.${call.member} is not a method`, - ); - } - const result: unknown = await Reflect.apply(member.method, instance.implementation, [...call.args, context]); - return result as JsonValue | undefined; - } - - subscribe( - serviceId: string, - mode: ServiceMode, - listener: (update: ServiceProviderUpdate, context: Context) => void, - ): ServiceSubscription { - this.#assertActive(); - this.#assertAllowed(serviceId); - const registration = this.#registration(serviceId, mode); - if (registration.mode === "singleton" && registration.singleton === undefined) { - throw new RemoteServiceError("service_not_found", `Remote service ${serviceId} has no provider`); - } - const subscriber: ProviderSubscriber = { - listener, - buffer: [], - snapshotSequences: new Map(), - active: false, - draining: false, - terminated: false, - closed: false, - }; - registration.subscribers.add(subscriber); - const snapshot = this.#snapshot(registration); - recordSnapshotSequences(subscriber.snapshotSequences, snapshot.instances); - return { - snapshot, - activate: () => { - if (subscriber.closed || subscriber.active) return; - subscriber.active = true; - throwCollectedErrors(drainSubscriber(subscriber), "Failed to activate remote service subscription"); - }, - close: () => { - if (subscriber.closed) return; - subscriber.closed = true; - subscriber.buffer.length = 0; - registration.subscribers.delete(subscriber); - }, - }; - } - - dispose(): void { - if (this.#disposed) return; - this.#disposed = true; - const errors: unknown[] = []; - for (const registration of this.#registrations.values()) { - const singleton = registration.singleton; - if (singleton !== undefined) { - singleton.active = false; - for (const remove of singleton.removeMemberListeners) remove(); - delete registration.singleton; - try { - this.#emit(registration, { type: "unavailable" }); - } catch (error) { - errors.push(error); - } - } - for (const [key, instance] of [...registration.instances]) { - instance.active = false; - for (const remove of instance.removeMemberListeners) remove(); - registration.instances.delete(key); - try { - this.#emit(registration, { type: "closed", instance: instance.address! }); - } catch (error) { - errors.push(error); - } - } - for (const subscriber of registration.subscribers) { - if (subscriber.active && !subscriber.draining) { - subscriber.closed = true; - subscriber.buffer.length = 0; - } else { - subscriber.terminated = true; - } - } - registration.subscribers.clear(); - } - this.#registrations.clear(); - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, "Failed to dispose remote service provider"); - } - - #registration(serviceId: string, mode: ServiceMode): ServiceRegistration { - const registration = this.#registrations.get(serviceId); - if (registration === undefined) { - throw new RemoteServiceError("service_not_found", `Unknown remote service ${serviceId}`); - } - if (registration.mode !== mode) { - throw new RemoteServiceError( - "service_mode_mismatch", - `Remote service ${serviceId} is ${registration.mode}, not ${mode}`, - ); - } - return registration; - } - - #createInstance( - registration: ServiceRegistration, - classified: ClassifiedRemoteServiceImplementation, - address: ServiceInstanceAddress | undefined, - ): ProviderInstance { - const removeMemberListeners: (() => void)[] = []; - const instance: ProviderInstance = { - ...(address === undefined ? {} : { address }), - implementation: classified.implementation, - members: classified.members, - removeMemberListeners, - active: true, - }; - for (const [name, member] of classified.members) { - if (member.kind !== "state") continue; - removeMemberListeners.push( - member.state.subscribe((ops, sequence, context) => { - if (!instance.active) return; - this.#emit( - registration, - { - type: "state", - ...(address === undefined ? {} : { instance: address }), - member: name, - sequence, - ops, - }, - context, - ); - }), - ); - } - return instance; - } - - #assertSingletonShape(registration: ServiceRegistration, replacement: ReadonlyMap): void { - const current = registration.singletonShape; - if (current === undefined || sameServiceMemberShape(current, replacement)) return; - throw new RemoteServiceError( - "service_member_mismatch", - `Remote service ${registration.serviceId} replacement must preserve its member shape`, - ); - } - - #resolveInstance(registration: ServiceRegistration, address: ServiceInstanceAddress | undefined): ProviderInstance { - if (registration.mode === "singleton") { - if (address !== undefined) { - throw new RemoteServiceError( - "service_mode_mismatch", - `Remote service ${registration.serviceId} is singleton`, - ); - } - if (registration.singleton === undefined) { - throw new RemoteServiceError( - "service_not_found", - `Remote service ${registration.serviceId} has no provider`, - ); - } - return registration.singleton; - } - if (address === undefined) { - throw new RemoteServiceError("service_mode_mismatch", `Remote service ${registration.serviceId} is keyed`); - } - const instance = registration.instances.get(address.key); - if (instance === undefined) { - throw new RemoteServiceError( - "service_instance_not_found", - `Remote service ${registration.serviceId} has no instance ${address.key}`, - ); - } - if (instance.address?.generation !== address.generation) { - throw new RemoteServiceError( - "service_stale_instance", - `Remote service ${registration.serviceId} instance ${address.key} is stale`, - ); - } - return instance; - } - - #snapshot(registration: ServiceRegistration): ServiceSubscriptionSnapshot { - const instances = - registration.mode === "singleton" - ? registration.singleton - ? [this.#snapshotInstance(registration.singleton)] - : [] - : [...registration.instances.values()] - .sort((left, right) => left.address!.key.localeCompare(right.address!.key)) - .map((instance) => this.#snapshotInstance(instance)); - return { serviceId: registration.serviceId, mode: registration.mode, instances }; - } - - #snapshotInstance(instance: ProviderInstance): ServiceInstanceSnapshot { - const members: ServiceMemberSnapshot[] = []; - for (const [name, member] of instance.members) { - if (member.kind === "method") { - members.push({ name, kind: "method" }); - } else { - const snapshot = member.state.snapshot(); - members.push({ - name, - kind: "state", - sequence: snapshot.sequence, - ops: [["r", snapshot.value as JsonValue]], - }); - } - } - return { - ...(instance.address === undefined ? {} : { instance: instance.address }), - members, - }; - } - - #emit(registration: ServiceRegistration, update: ServiceProviderUpdate, context?: Context): void { - if (registration.subscribers.size === 0) return; - const deliveryContext = context ?? serviceDeliveryContext(); - const errors: unknown[] = []; - const subscribers = [...registration.subscribers]; - // Queue for everyone before invoking user code, including reentrant publications. - for (const subscriber of subscribers) { - if (subscriber.closed || updateCoveredBySnapshot(subscriber.snapshotSequences, update)) continue; - if (subscriber.buffer.length === 100) { - const snapshot = this.#snapshot(registration); - subscriber.buffer.length = 0; - subscriber.snapshotSequences.clear(); - recordSnapshotSequences(subscriber.snapshotSequences, snapshot.instances); - subscriber.buffer.push({ update: { type: "reset", snapshot }, context: deliveryContext }); - } else { - subscriber.buffer.push({ update, context: deliveryContext }); - } - } - for (const subscriber of subscribers) errors.push(...drainSubscriber(subscriber)); - throwCollectedErrors(errors, `Failed to publish remote service ${registration.serviceId} update`); - } - - #assertRemotable(service: { readonly id: string; readonly local: boolean }): void { - if (service.local) throw new RemoteServiceError("service_not_allowed", `Service ${service.id} is process-local`); - } - - #assertAllowed(serviceId: string): void { - if (!this.#registrations.has(serviceId)) { - throw new RemoteServiceError("service_not_allowed", `Remote service ${serviceId} is not allowlisted`); - } - } - - #assertActive(): void { - if (this.#disposed) throw new Error("Remote service provider is disposed"); - } -} - -function drainSubscriber(subscriber: ProviderSubscriber): unknown[] { - if (!subscriber.active || subscriber.closed || subscriber.draining) return []; - const errors: unknown[] = []; - subscriber.draining = true; - try { - while (!subscriber.closed) { - const entry = subscriber.buffer.shift(); - if (entry === undefined) break; - try { - subscriber.listener(entry.update, entry.context); - } catch (error) { - errors.push(error); - } - } - } finally { - subscriber.draining = false; - if (subscriber.terminated) subscriber.closed = true; - } - return errors; -} - -function recordSnapshotSequences(sequences: Map, instances: readonly ServiceInstanceSnapshot[]): void { - for (const instance of instances) { - for (const member of instance.members) { - if (member.kind === "state") sequences.set(stateMemberKey(instance.instance, member.name), member.sequence); - } - } -} - -function updateCoveredBySnapshot(sequences: Map, update: ServiceProviderUpdate): boolean { - switch (update.type) { - case "state": { - const key = stateMemberKey(update.instance, update.member); - const sequence = sequences.get(key); - if (sequence === undefined) return false; - if (update.sequence <= sequence) return true; - sequences.delete(key); - return false; - } - case "reset": - sequences.clear(); - recordSnapshotSequences(sequences, update.snapshot.instances); - return false; - case "replaced": - sequences.clear(); - recordSnapshotSequences(sequences, [update.snapshot]); - return false; - case "spawned": - recordSnapshotSequences(sequences, [update.instance]); - return false; - case "unavailable": - sequences.clear(); - return false; - case "closed": { - const prefix = `${JSON.stringify([update.instance.key, update.instance.generation]).slice(0, -1)},`; - for (const key of sequences.keys()) { - if (key.startsWith(prefix)) sequences.delete(key); - } - return false; - } - } -} - -function stateMemberKey(instance: ServiceInstanceAddress | undefined, member: string): string { - return JSON.stringify(instance === undefined ? [member] : [instance.key, instance.generation, member]); -} - -export function createRemoteServiceEndpoint(provider: RemoteServiceProvider): RemoteServiceEndpoint { - const subscriptions = new Map(); - let disposed = false; - - return { - async invoke(call, publish, context) { - if (disposed) throw new Error("Remote service endpoint is disposed"); - const control = decodeServiceControlCall(call); - if (control?.type === "catalogue") return provider.catalogue as unknown as JsonValue; - if (control?.type === "subscribe") { - if (subscriptions.has(control.subscriptionId)) { - throw new Error("Service subscription ID is already active"); - } - const subscription = provider.subscribe(control.serviceId, control.mode, (update, updateContext) => { - void Promise.resolve(publish(control.subscriptionId, update, updateContext)).catch(() => {}); - }); - subscriptions.set(control.subscriptionId, subscription); - subscription.activate(); - return subscription.snapshot as unknown as JsonValue; - } - if (control?.type === "unsubscribe") { - const subscription = subscriptions.get(control.subscriptionId); - if (subscription === undefined) throw new Error("Service subscription was not found"); - subscription.close(); - subscriptions.delete(control.subscriptionId); - return undefined; - } - return provider.invoke(call, context); - }, - dispose() { - if (disposed) return; - disposed = true; - for (const subscription of subscriptions.values()) subscription.close(); - subscriptions.clear(); - }, - }; -} - -export function validateRemoteServiceImplementation(serviceId: string, implementation: unknown): void { - void classifyRemoteServiceImplementation(serviceId, implementation); -} - -function classifyRemoteServiceImplementation( - serviceId: string, - implementation: unknown, -): ClassifiedRemoteServiceImplementation { - if (typeof implementation !== "object" || implementation === null || Array.isArray(implementation)) { - throw new TypeError(`Remote service ${serviceId} implementation must be an object`); - } - const members = new Map(); - for (const name of Object.keys(implementation).sort()) { - const descriptor = Object.getOwnPropertyDescriptor(implementation, name); - if (descriptor === undefined || !("value" in descriptor)) { - throw new TypeError(`Remote service member ${serviceId}.${name} must be a data property`); - } - if (typeof descriptor.value === "function") { - members.set(name, { kind: "method", method: descriptor.value as RemoteMethod }); - continue; - } - const state = getReplicatedStateInternals(descriptor.value); - if (state !== undefined) { - members.set(name, { kind: "state", state }); - continue; - } - throw new TypeError(`Remote service member ${serviceId}.${name} is not remotely exposable`); - } - if (members.size === 0) throw new TypeError(`Remote service ${serviceId} has no members`); - return { implementation, members }; -} - -function throwCollectedErrors(errors: readonly unknown[], message: string): void { - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, message); -} - -function serviceMemberShape(members: ReadonlyMap): ReadonlyMap { - return new Map([...members].map(([name, member]) => [name, member.kind])); -} - -function sameServiceMemberShape( - left: ReadonlyMap, - right: ReadonlyMap, -): boolean { - return left.size === right.size && [...left].every(([name, kind]) => right.get(name) === kind); -} diff --git a/packages/chord/src/services/state-codec.ts b/packages/chord/src/services/state-codec.ts deleted file mode 100644 index f5e26722e..000000000 --- a/packages/chord/src/services/state-codec.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { type Decoder, decoder, type Encoder, encoder } from "../delta/index.ts"; -import type { - ServiceInstanceAddress, - ServiceInstanceSnapshot, - ServiceProviderUpdate, - ServiceSubscriptionSnapshot, -} from "../types.ts"; -import type { WireServiceProviderUpdate, WireServiceSubscriptionSnapshot } from "./wire.ts"; - -/** Stateful operation encoders for every replicated state in one service subscription. */ -export interface ServiceStateEncoder { - encodeSnapshot(snapshot: ServiceSubscriptionSnapshot): WireServiceSubscriptionSnapshot; - encodeUpdate(update: ServiceProviderUpdate): WireServiceProviderUpdate; -} - -/** Stateful operation decoders for every replicated state in one service subscription. */ -export interface ServiceStateDecoder { - decodeSnapshot(snapshot: WireServiceSubscriptionSnapshot): ServiceSubscriptionSnapshot; - decodeUpdate(update: WireServiceProviderUpdate): ServiceProviderUpdate; -} - -interface CodecEntry { - readonly instance: ServiceInstanceAddress | undefined; - readonly codec: C; -} - -class StateCodecRegistry { - readonly #create: () => C; - readonly #entries = new Map>(); - - constructor(create: () => C) { - this.#create = create; - } - - reset(): void { - this.#entries.clear(); - } - - add(instance: ServiceInstanceAddress | undefined, member: string): C { - const key = stateKey(instance, member); - if (this.#entries.has(key)) throw new Error(`Duplicate service state ${describeState(instance, member)}`); - const codec = this.#create(); - this.#entries.set(key, { instance, codec }); - return codec; - } - - get(instance: ServiceInstanceAddress | undefined, member: string): C { - const codec = this.#entries.get(stateKey(instance, member))?.codec; - if (codec === undefined) throw new Error(`Unknown service state ${describeState(instance, member)}`); - return codec; - } - - removeInstance(instance: ServiceInstanceAddress): void { - for (const [key, entry] of this.#entries) { - if (sameAddress(entry.instance, instance)) this.#entries.delete(key); - } - } -} - -export function createServiceStateEncoder(): ServiceStateEncoder { - const codecs = new StateCodecRegistry(encoder); - return { - encodeSnapshot(snapshot) { - codecs.reset(); - return { - ...snapshot, - instances: snapshot.instances.map((instance) => encodeInstance(instance, codecs)), - }; - }, - encodeUpdate(update) { - switch (update.type) { - case "reset": - codecs.reset(); - return { - ...update, - snapshot: { - ...update.snapshot, - instances: update.snapshot.instances.map((instance) => encodeInstance(instance, codecs)), - }, - }; - case "state": - return { ...update, ops: codecs.get(update.instance, update.member).encode(update.ops) }; - case "replaced": - codecs.reset(); - return { ...update, snapshot: encodeInstance(update.snapshot, codecs) }; - case "spawned": - return { ...update, instance: encodeInstance(update.instance, codecs) }; - case "unavailable": - codecs.reset(); - return update; - case "closed": - codecs.removeInstance(update.instance); - return update; - } - }, - }; -} - -export function createServiceStateDecoder(): ServiceStateDecoder { - const codecs = new StateCodecRegistry(decoder); - return { - decodeSnapshot(snapshot) { - codecs.reset(); - return { - ...snapshot, - instances: snapshot.instances.map((instance) => decodeInstance(instance, codecs)), - }; - }, - decodeUpdate(update) { - switch (update.type) { - case "reset": - codecs.reset(); - return { - ...update, - snapshot: { - ...update.snapshot, - instances: update.snapshot.instances.map((instance) => decodeInstance(instance, codecs)), - }, - }; - case "state": - return { ...update, ops: codecs.get(update.instance, update.member).decode(update.ops) }; - case "replaced": - codecs.reset(); - return { ...update, snapshot: decodeInstance(update.snapshot, codecs) }; - case "spawned": - return { ...update, instance: decodeInstance(update.instance, codecs) }; - case "unavailable": - codecs.reset(); - return update; - case "closed": - codecs.removeInstance(update.instance); - return update; - } - }, - }; -} - -function encodeInstance( - instance: ServiceInstanceSnapshot, - codecs: StateCodecRegistry, -): WireServiceSubscriptionSnapshot["instances"][number] { - return { - ...instance, - members: instance.members.map((member) => - member.kind === "state" - ? { ...member, ops: codecs.add(instance.instance, member.name).encode(member.ops) } - : member, - ), - }; -} - -function decodeInstance( - instance: WireServiceSubscriptionSnapshot["instances"][number], - codecs: StateCodecRegistry, -): ServiceInstanceSnapshot { - return { - ...instance, - members: instance.members.map((member) => - member.kind === "state" - ? { ...member, ops: codecs.add(instance.instance, member.name).decode(member.ops) } - : member, - ), - }; -} - -function stateKey(instance: ServiceInstanceAddress | undefined, member: string): string { - return JSON.stringify([instance?.key ?? null, instance?.generation ?? null, member]); -} - -function sameAddress(left: ServiceInstanceAddress | undefined, right: ServiceInstanceAddress): boolean { - return left?.key === right.key && left.generation === right.generation; -} - -function describeState(instance: ServiceInstanceAddress | undefined, member: string): string { - return instance === undefined ? member : `${instance.key}@${instance.generation}.${member}`; -} diff --git a/packages/chord/src/services/state-internals.ts b/packages/chord/src/services/state-internals.ts deleted file mode 100644 index ed5c1ff0e..000000000 --- a/packages/chord/src/services/state-internals.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { Op } from "../delta/index.ts"; -import type { Context } from "../types.ts"; - -export interface ReplicatedStateInternals { - /** Atomically capture the immutable value and its matching publication sequence. */ - snapshot(): { readonly value: unknown; readonly sequence: number }; - subscribe(listener: (ops: readonly Op[], sequence: number, context: Context) => void): () => void; -} - -const sources = new WeakMap(); - -export function registerReplicatedStateInternals(value: object, internals: ReplicatedStateInternals): void { - sources.set(value, internals); -} - -export function getReplicatedStateInternals(value: unknown): ReplicatedStateInternals | undefined { - if (typeof value !== "object" || value === null) return undefined; - return sources.get(value); -} diff --git a/packages/chord/src/services/wire.ts b/packages/chord/src/services/wire.ts deleted file mode 100644 index 370935c13..000000000 --- a/packages/chord/src/services/wire.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { assertValidOp, assertValidWireOp, type WireOp } from "../delta/index.ts"; -import type { - ServiceCall, - ServiceCatalogueEntry, - ServiceInstanceAddress, - ServiceMode, - ServiceProviderUpdate, - ServiceSubscriptionSnapshot, -} from "../types.ts"; - -export type WireServiceMemberSnapshot = - | { readonly name: string; readonly kind: "method" } - | { readonly name: string; readonly kind: "state"; readonly sequence: number; readonly ops: readonly WireOp[] }; - -export type WireServiceInstanceSnapshot = { - readonly instance?: ServiceInstanceAddress; - readonly members: readonly WireServiceMemberSnapshot[]; -}; - -export type WireServiceSubscriptionSnapshot = { - readonly serviceId: string; - readonly mode: ServiceMode; - readonly instances: readonly WireServiceInstanceSnapshot[]; -}; - -export type WireServiceProviderUpdate = - | { - readonly type: "state"; - readonly instance?: ServiceInstanceAddress; - readonly member: string; - readonly sequence: number; - readonly ops: readonly WireOp[]; - } - | { readonly type: "reset"; readonly snapshot: WireServiceSubscriptionSnapshot } - | { readonly type: "unavailable" } - | { readonly type: "replaced"; readonly snapshot: WireServiceInstanceSnapshot } - | { readonly type: "spawned"; readonly instance: WireServiceInstanceSnapshot } - | { readonly type: "closed"; readonly instance: ServiceInstanceAddress }; - -const SERVICE_CONTROL_ID = "$chord.service"; -const SERVICE_CATALOGUE_MEMBER = "catalogue"; -const SERVICE_SUBSCRIBE_MEMBER = "subscribe"; -const SERVICE_UNSUBSCRIBE_MEMBER = "unsubscribe"; - -export type ServiceControlCall = - | { readonly type: "catalogue" } - | { - readonly type: "subscribe"; - readonly subscriptionId: string; - readonly serviceId: string; - readonly mode: ServiceMode; - } - | { readonly type: "unsubscribe"; readonly subscriptionId: string }; - -export function createServiceCatalogueCall(): ServiceCall { - return { serviceId: SERVICE_CONTROL_ID, member: SERVICE_CATALOGUE_MEMBER, args: [] }; -} - -export function createServiceSubscribeCall(subscriptionId: string, serviceId: string, mode: ServiceMode): ServiceCall { - return { serviceId: SERVICE_CONTROL_ID, member: SERVICE_SUBSCRIBE_MEMBER, args: [subscriptionId, serviceId, mode] }; -} - -export function createServiceUnsubscribeCall(subscriptionId: string): ServiceCall { - return { serviceId: SERVICE_CONTROL_ID, member: SERVICE_UNSUBSCRIBE_MEMBER, args: [subscriptionId] }; -} - -export function decodeServiceControlCall(call: ServiceCall): ServiceControlCall | undefined { - if (call.serviceId !== SERVICE_CONTROL_ID || call.instance !== undefined) return undefined; - if (call.member === SERVICE_CATALOGUE_MEMBER && call.args.length === 0) return { type: "catalogue" }; - if ( - call.member === SERVICE_SUBSCRIBE_MEMBER && - call.args.length === 3 && - isId(call.args[0]) && - isId(call.args[1]) && - (call.args[2] === "singleton" || call.args[2] === "keyed") - ) { - return { - type: "subscribe", - subscriptionId: call.args[0], - serviceId: call.args[1], - mode: call.args[2], - }; - } - if (call.member === SERVICE_UNSUBSCRIBE_MEMBER && call.args.length === 1 && isId(call.args[0])) { - return { type: "unsubscribe", subscriptionId: call.args[0] }; - } - return undefined; -} - -export function parseServiceCall(value: unknown): ServiceCall { - const call = record(value, "service call"); - assertKeys(call, ["serviceId", "member", "args"], ["instance"], "service call"); - if (!isId(call.serviceId) || !isId(call.member) || !Array.isArray(call.args)) { - throw new TypeError("Invalid service call"); - } - if (call.instance !== undefined) assertAddress(call.instance); - return value as ServiceCall; -} - -export function parseServiceCatalogue(value: unknown): readonly ServiceCatalogueEntry[] { - if (!Array.isArray(value)) throw new TypeError("Invalid service catalogue"); - const ids = new Set(); - for (const candidate of value) { - const entry = record(candidate, "service catalogue entry"); - assertKeys(entry, ["serviceId", "mode"], [], "service catalogue entry"); - if (!isId(entry.serviceId) || !isMode(entry.mode) || ids.has(entry.serviceId)) { - throw new TypeError("Invalid service catalogue"); - } - ids.add(entry.serviceId); - } - return value as unknown as readonly ServiceCatalogueEntry[]; -} - -export function parseServiceSubscriptionSnapshot(value: unknown): ServiceSubscriptionSnapshot { - assertSubscriptionSnapshot(value, assertValidOp); - return value as ServiceSubscriptionSnapshot; -} - -export function parseWireServiceSubscriptionSnapshot(value: unknown): WireServiceSubscriptionSnapshot { - assertSubscriptionSnapshot(value, assertValidWireOp); - return value as WireServiceSubscriptionSnapshot; -} - -export function parseServiceProviderUpdate(value: unknown): ServiceProviderUpdate { - assertProviderUpdate(value, assertValidOp); - return value as ServiceProviderUpdate; -} - -export function parseWireServiceProviderUpdate(value: unknown): WireServiceProviderUpdate { - assertProviderUpdate(value, assertValidWireOp); - return value as WireServiceProviderUpdate; -} - -function assertSubscriptionSnapshot(value: unknown, assertOp: (value: unknown) => void): void { - const snapshot = record(value, "service subscription snapshot"); - assertKeys(snapshot, ["serviceId", "mode", "instances"], [], "service subscription snapshot"); - if (!isId(snapshot.serviceId) || !isMode(snapshot.mode) || !Array.isArray(snapshot.instances)) { - throw new TypeError("Invalid service subscription snapshot"); - } - for (const instance of snapshot.instances) assertInstance(instance, assertOp); -} - -function assertProviderUpdate(value: unknown, assertOp: (value: unknown) => void): void { - const update = record(value, "service provider update"); - switch (update.type) { - case "state": - assertKeys(update, ["type", "member", "sequence", "ops"], ["instance"], "state update"); - if (!isId(update.member) || !isInteger(update.sequence, 1) || !Array.isArray(update.ops)) { - throw new TypeError("Invalid service state update"); - } - if (update.instance !== undefined) assertAddress(update.instance); - for (const op of update.ops) assertOp(op); - return; - case "reset": { - assertKeys(update, ["type", "snapshot"], [], "reset update"); - assertSubscriptionSnapshot(update.snapshot, assertOp); - const snapshot = update.snapshot as WireServiceSubscriptionSnapshot; - for (const instance of snapshot.instances) { - for (const member of instance.members) { - if (member.kind === "state" && (member.ops.length !== 1 || member.ops[0]?.[0] !== "r")) { - throw new TypeError("Service reset must contain full root replacements"); - } - } - } - return; - } - case "unavailable": - assertKeys(update, ["type"], [], "unavailable update"); - return; - case "replaced": - assertKeys(update, ["type", "snapshot"], [], "replacement update"); - assertInstance(update.snapshot, assertOp); - return; - case "spawned": - assertKeys(update, ["type", "instance"], [], "spawn update"); - assertInstance(update.instance, assertOp); - return; - case "closed": - assertKeys(update, ["type", "instance"], [], "close update"); - assertAddress(update.instance); - return; - default: - throw new TypeError("Invalid service provider update"); - } -} - -function assertInstance(value: unknown, assertOp: (value: unknown) => void): void { - const instance = record(value, "service instance snapshot"); - assertKeys(instance, ["members"], ["instance"], "service instance snapshot"); - if (instance.instance !== undefined) assertAddress(instance.instance); - if (!Array.isArray(instance.members)) throw new TypeError("Invalid service instance snapshot"); - for (const candidate of instance.members) { - const member = record(candidate, "service member snapshot"); - if (member.kind === "method") { - assertKeys(member, ["name", "kind"], [], "service method snapshot"); - if (!isId(member.name)) throw new TypeError("Invalid service method snapshot"); - continue; - } - if (member.kind === "state") { - assertKeys(member, ["name", "kind", "sequence", "ops"], [], "service state snapshot"); - if (!isId(member.name) || !isInteger(member.sequence, 0) || !Array.isArray(member.ops)) { - throw new TypeError("Invalid service state snapshot"); - } - for (const op of member.ops) assertOp(op); - continue; - } - throw new TypeError("Invalid service member snapshot"); - } -} - -function assertAddress(value: unknown): asserts value is ServiceInstanceAddress { - const address = record(value, "service instance address"); - assertKeys(address, ["key", "generation"], [], "service instance address"); - if (!isId(address.key) || !isInteger(address.generation, 1)) { - throw new TypeError("Invalid service instance address"); - } -} - -function record(value: unknown, description: string): Record { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - throw new TypeError(`Invalid ${description}`); - } - return value as Record; -} - -function assertKeys( - value: Record, - required: readonly string[], - optional: readonly string[], - description: string, -): void { - const allowed = new Set([...required, ...optional]); - if (required.some((key) => !Object.hasOwn(value, key)) || Object.keys(value).some((key) => !allowed.has(key))) { - throw new TypeError(`Invalid ${description}`); - } -} - -function isId(value: unknown): value is string { - return typeof value === "string" && value.length > 0; -} - -function isMode(value: unknown): value is ServiceMode { - return value === "singleton" || value === "keyed"; -} - -function isInteger(value: unknown, minimum: number): value is number { - return typeof value === "number" && Number.isInteger(value) && value >= minimum; -} diff --git a/packages/chord/src/services/state.ts b/packages/chord/src/state.ts similarity index 57% rename from packages/chord/src/services/state.ts rename to packages/chord/src/state.ts index 505ac69bd..08b4ed107 100644 --- a/packages/chord/src/services/state.ts +++ b/packages/chord/src/state.ts @@ -1,19 +1,93 @@ -import { BACKGROUND_CONTEXT } from "../context/index.ts"; -import { applyImmutable, isBase, type Op, type Prepared, type Tracker, track } from "../delta/index.ts"; -import { JsonRevisionValidator } from "../delta/revision-validator.ts"; -import type { - AttachedReplicatedState, - Context, - JsonValue, - MutableReplicatedState, - ReplicatedState, - ReplicatedStateDelivery, - ReplicatedStateSource, - ReplicatedStateSourceAttachment, - ReplicatedStateSourceFrame, - ReplicatedStateSourceOptions, -} from "../types.ts"; -import { registerReplicatedStateInternals } from "./state-internals.ts"; +/** + * Replicated state: an immutable JSON value with ordered, serialized change notifications. Either authoritative and + * mutable (`replicatedState(initial)`) or a publication-only view of an external revision source + * (`replicatedState(source)`). + */ +import { BACKGROUND_CONTEXT, type Context } from "./context/index.ts"; +import { type Draft, type Op, type Prepared, type Tracker, track } from "./delta/index.ts"; + +export interface ReplicatedStateDelivery { + readonly kind: "hydrate" | "update"; + readonly sequence: number; +} + +export interface ReplicatedState { + /** + * Contract-immutable value, or undefined until hydration. It is not frozen and may share containers with an + * in-process provider; consumers must not mutate it. Later updates do not mutate previously returned values. + */ + readonly value: T | undefined; + /** + * Each subscription serializes callbacks, awaiting hydration before updates. Values are immutable and may + * share unchanged revision data. At most 100 deliveries wait behind the running callback; overflow keeps + * only the newest pending value/context/delivery, so update sequences may skip. Failures are reported in + * isolation and delivery continues. Unsubscribe discards pending work without aborting or joining a callback. + */ + subscribe(listener: (value: T, context: Context, delivery: ReplicatedStateDelivery) => Promise): () => void; + subscribe(listener: (value: T, context: Context, delivery: ReplicatedStateDelivery) => void): () => void; +} + +export interface MutableReplicatedState extends ReplicatedState { + readonly value: T; + /** + * Atomically publish one synchronous overlay mutation. Draft handles are unusable after the callback returns. + * Values placed through the draft are cloned by value; assigning `undefined` to an object property deletes it. + */ + change(context: Context, mutate: (draft: Draft) => void): void; + /** + * Atomically take immutable ownership of an alias-free strict-JSON replacement. + * The caller must not mutate the transferred root after this call. + */ + replace(context: Context, value: T): void; +} + +/** One immutable authoritative revision committed after an attachment snapshot. */ +export interface ReplicatedStateSourceFrame { + /** Monotonic source cursor. The first frame after a snapshot must be `snapshot.cursor + 1`. */ + readonly cursor: number; + /** The exact immutable value produced by this commit. */ + readonly value: T; + /** The exact immutable operation batch that produced `value` from the preceding source revision. */ + readonly ops: readonly Op[]; + readonly context: Context; +} + +export interface ReplicatedStateSourceAttachment { + /** Fixed immutable snapshot captured at the atomic attachment boundary. */ + readonly snapshot: { readonly value: T; readonly cursor: number }; + /** + * Install the sole listener and synchronously drain every buffered frame in source commit order. + * This method is single-use. After it begins, every new committed frame must also be delivered in order + * until disposal, including commits made reentrantly while a prior frame is being delivered. + */ + activate(listener: (frame: ReplicatedStateSourceFrame) => void): void; + /** Stop delivery and release source resources. Disposal must be idempotent. */ + dispose(): void; +} + +/** + * An authoritative immutable revision source. + * + * `attach()` must synchronously and atomically capture one snapshot and register the returned attachment to buffer + * every later committed frame. The snapshot must include every commit before that boundary; buffered frames must + * include every commit after it, with no overlap or gap. Snapshot values, frame values, and operation batches are + * immutable and remain valid after delivery. Chord only publishes these references; it never applies or re-diffs them. + */ +export interface ReplicatedStateSource { + attach(): ReplicatedStateSourceAttachment; +} + +export interface ReplicatedStateSourceOptions { + /** Receives source-contract and publication-listener failures without throwing them into the source. */ + readonly onError?: (error: Error) => void; +} + +/** A synchronously hydrated publication-only state backed by one source attachment. */ +export interface AttachedReplicatedState extends ReplicatedState { + readonly value: T; + /** Idempotently release the source attachment. The last published value remains readable. */ + dispose(): void; +} /** The void signature also accepts synchronous callbacks that return an ignored value. */ type StateListener = (value: T, context: Context, delivery: ReplicatedStateDelivery) => void; @@ -96,11 +170,8 @@ class StateSubscriber { } } -type SourceListener = (ops: readonly Op[], sequence: number, context: Context) => void; - type Publication = { readonly value: T; - readonly ops: readonly Op[]; readonly sequence: number; readonly context: Context; }; @@ -109,7 +180,6 @@ type Publication = { class ReplicatedStatePublisher { readonly #listeners = new Map, number>(); readonly #reportError: (error: Error) => void; - readonly #sourceListeners = new Set(); readonly #publications: Publication[] = []; #value: T; #sequence = 0; @@ -132,7 +202,8 @@ class ReplicatedStatePublisher { const { value, sequence } = this.snapshot(); const subscriber = new StateSubscriber(listener, this.#reportError); this.#listeners.set(subscriber, sequence); - subscriber.push({ value, context: serviceDeliveryContext(), delivery: { kind: "hydrate", sequence } }); + // Hydration has no caller; it is delivered with the background context. + subscriber.push({ value, context: BACKGROUND_CONTEXT, delivery: { kind: "hydrate", sequence } }); subscriber.drain(); return () => { subscriber.close(); @@ -140,33 +211,20 @@ class ReplicatedStatePublisher { }; } - subscribeSource(listener: SourceListener): () => void { - this.#sourceListeners.add(listener); - return () => this.#sourceListeners.delete(listener); - } - - /** Publish an already-prepared immutable revision and return isolated listener failures. */ - publish(value: T, ops: readonly Op[], context: Context): unknown[] { + /** Publish an already-prepared immutable revision. Subscriber failures are reported, never thrown. */ + publish(value: T, context: Context): void { this.#value = value; this.#sequence += 1; - this.#publications.push({ value, ops, sequence: this.#sequence, context }); - if (this.#delivering) return []; + this.#publications.push({ value, sequence: this.#sequence, context }); + if (this.#delivering) return; this.#delivering = true; - const errors: unknown[] = []; try { for ( let publication = this.#publications.shift(); publication !== undefined; publication = this.#publications.shift() ) { - for (const listener of [...this.#sourceListeners]) { - try { - listener(publication.ops, publication.sequence, publication.context); - } catch (error) { - errors.push(error); - } - } const delivery = { kind: "update", sequence: publication.sequence } as const; for (const [subscriber, hydratedSequence] of [...this.#listeners]) { if (publication.sequence <= hydratedSequence) continue; @@ -177,11 +235,10 @@ class ReplicatedStatePublisher { } finally { this.#delivering = false; } - return errors; } } -export class MutableReplicatedStateImpl implements MutableReplicatedState { +class MutableReplicatedStateImpl implements MutableReplicatedState { readonly #tracker: Tracker; readonly #publisher: ReplicatedStatePublisher; #changing = false; @@ -189,10 +246,6 @@ export class MutableReplicatedStateImpl implements MutableRepl constructor(initial: T) { this.#tracker = track(initial); this.#publisher = new ReplicatedStatePublisher(this.#tracker.value); - registerReplicatedStateInternals(this, { - snapshot: () => this.#publisher.snapshot(), - subscribe: (listener) => this.#publisher.subscribeSource(listener), - }); } get value(): T { @@ -221,22 +274,14 @@ export class MutableReplicatedStateImpl implements MutableRepl } this.#tracker.adopt(prepared); - if (prepared.ops.length === 0) return; - throwCollectedErrors( - this.#publisher.publish(prepared.value, prepared.ops, context), - "Replicated state listeners failed", - ); + if (prepared.ops.length > 0) this.#publisher.publish(prepared.value, context); } replace(context: Context, value: T): void { if (this.#changing) throw new Error("Replicated state cannot be replaced from a change callback"); const prepared = this.#tracker.prepareReplace(value); this.#tracker.adopt(prepared); - if (prepared.ops.length === 0) return; - throwCollectedErrors( - this.#publisher.publish(prepared.value, prepared.ops, context), - "Replicated state listeners failed", - ); + if (prepared.ops.length > 0) this.#publisher.publish(prepared.value, context); } subscribe(listener: StateListener): () => void { @@ -258,10 +303,6 @@ class AttachedReplicatedStateImpl implements AttachedReplicatedState { this.#cursor = snapshot.cursor; this.#reportError = options.onError ?? reportErrorAsync; this.#publisher = new ReplicatedStatePublisher(snapshot.value, (error) => this.#report(error)); - registerReplicatedStateInternals(this, { - snapshot: () => this.#publisher.snapshot(), - subscribe: (listener) => this.#publisher.subscribeSource(listener), - }); } get value(): T { @@ -291,9 +332,7 @@ class AttachedReplicatedStateImpl implements AttachedReplicatedState { throw new Error(`Replicated state source cursor has a gap: expected ${expected}, received ${frame.cursor}`); } this.#cursor = frame.cursor; - const errors = this.#publisher.publish(frame.value, frame.ops, frame.context); - if (errors.length === 1) this.#report(errors[0]); - else if (errors.length > 1) this.#report(new AggregateError(errors, "Replicated state listeners failed")); + this.#publisher.publish(frame.value, frame.context); } catch (error) { this.#fail(toError(error)); } @@ -321,7 +360,7 @@ class AttachedReplicatedStateImpl implements AttachedReplicatedState { } /** Attach a publication-only replicated state to one authoritative immutable source stream. */ -export function attachReplicatedStateSource( +function attachReplicatedStateSource( source: ReplicatedStateSource, options: ReplicatedStateSourceOptions = {}, ): AttachedReplicatedState { @@ -340,95 +379,6 @@ export function attachReplicatedStateSource( } } -/** A cold read-only state used by service consumers until a complete snapshot arrives. */ -export class ReplicatedStateReplica implements ReplicatedState { - readonly #listeners = new Set>(); - readonly #reportError: (error: Error) => void; - readonly #validator = new JsonRevisionValidator(); - #value: T | undefined; - #sequence: number | undefined; - - constructor(reportError: (error: Error) => void) { - this.#reportError = reportError; - } - - get value(): T | undefined { - return this.#value; - } - - subscribe(listener: StateListener): () => void { - const subscriber = new StateSubscriber(listener, this.#reportError); - this.#listeners.add(subscriber); - if (this.#value !== undefined) { - subscriber.push({ - value: this.#value, - context: serviceDeliveryContext(), - delivery: { kind: "hydrate", sequence: this.#sequence! }, - }); - subscriber.drain(); - } - return () => { - subscriber.close(); - this.#listeners.delete(subscriber); - }; - } - - hydrate(sequence: number, ops: readonly Op[], context: Context): void { - let next: T; - try { - if (!isBase(ops)) throw new Error("Replicated state snapshot is not a base operation batch"); - next = this.#validator.validate(applyImmutable(undefined, ops)); - } catch (error) { - this.clear(); - throw error; - } - this.#sequence = sequence; - this.#value = next; - this.#deliverAll(context, { kind: "hydrate", sequence }); - } - - update(sequence: number, ops: readonly Op[], context: Context): void { - if (this.#sequence === undefined || this.#value === undefined) { - throw new Error("Replicated state received an update before hydration"); - } - if (sequence !== this.#sequence + 1) { - this.clear(); - throw new Error("Replicated state update sequence has a gap"); - } - let next: T; - try { - next = this.#validator.validate(applyImmutable(this.#value, ops)); - } catch (error) { - this.clear(); - throw error; - } - this.#sequence = sequence; - this.#value = next; - this.#deliverAll(context, { kind: "update", sequence }); - } - - clear(): void { - this.#value = undefined; - this.#sequence = undefined; - for (const subscriber of this.#listeners) subscriber.clear(); - } - - #deliverAll(context: Context, delivery: ReplicatedStateDelivery): void { - if (this.#value === undefined) return; - const frame = { value: this.#value, context, delivery }; - const subscribers = [...this.#listeners]; - // Enqueue for everyone before user code can publish another revision reentrantly. - for (const subscriber of subscribers) subscriber.push(frame); - for (const subscriber of subscribers) subscriber.drain(); - } -} - -/** @internal Context for synthetic service deliveries without a caller. */ -export function serviceDeliveryContext(): Context { - // TODO: Add delivery-scoped cancellation or metadata if deliveries gain an owned lifecycle. - return BACKGROUND_CONTEXT; -} - function assertCursor(cursor: number, kind: "snapshot" | "frame"): void { if (!Number.isSafeInteger(cursor)) throw new TypeError(`Replicated state source ${kind} cursor must be a safe integer`); @@ -441,11 +391,6 @@ function isPromiseLike(value: unknown): value is PromiseLike { ); } -function throwCollectedErrors(errors: readonly unknown[], message: string): void { - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) throw new AggregateError(errors, message); -} - function reportErrorAsync(error: Error): void { queueMicrotask(() => { throw error; @@ -455,3 +400,24 @@ function reportErrorAsync(error: Error): void { function toError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } + +export function replicatedState( + source: ReplicatedStateSource, + options?: ReplicatedStateSourceOptions, +): AttachedReplicatedState; +/** + * Create authoritative state by taking immutable ownership of an alias-free strict-JSON root. + * The caller must not mutate `initial` after this call. + */ +export function replicatedState(initial: T): MutableReplicatedState; +export function replicatedState( + initialOrSource: object | ReplicatedStateSource, + options?: ReplicatedStateSourceOptions, +): MutableReplicatedState | AttachedReplicatedState { + if (isReplicatedStateSource(initialOrSource)) return attachReplicatedStateSource(initialOrSource, options); + return new MutableReplicatedStateImpl(initialOrSource); +} + +function isReplicatedStateSource(value: object): value is ReplicatedStateSource { + return typeof (value as { readonly attach?: unknown }).attach === "function"; +} diff --git a/packages/chord/src/types.ts b/packages/chord/src/types.ts index 19ac55e74..cb04296a1 100644 --- a/packages/chord/src/types.ts +++ b/packages/chord/src/types.ts @@ -1,339 +1,79 @@ -import type { Draft, Op } from "./delta/index.ts"; -import type { RemoteServiceProvider } from "./services/provider.ts"; +import type { AnyContract, Handle, InputOf } from "./contract.ts"; +import type { Op } from "./delta/index.ts"; -export type { RemoteServiceError } from "./services/errors.ts"; -export type { RemoteServiceProvider } from "./services/provider.ts"; +/** App-defined, JSON-serializable identity of a caller. Mapped to a record key by `createHost({ principalKey })`. */ +export type Principal = unknown; -/** Typed identity for one value carried by a {@link Context}. */ -export interface ContextKey { - readonly token: symbol; - /** Type-only marker that prevents keys with different value types from being interchangeable. */ - readonly valueType?: (value: T) => T; -} - -/** Immutable invocation-scoped values passed explicitly through operations. */ -export interface Context { - readonly abortSignal: AbortSignal | undefined; - value(key: ContextKey): T | undefined; - toString(): string; -} - -export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; - -type IsAny = 0 extends 1 & T ? true : false; - -/** Strict-JSON representation of an application data type. Unknown payloads become JsonValue. */ -export type JsonRepresentation = IsAny extends true - ? JsonValue - : unknown extends T - ? JsonValue - : T extends null | boolean | number | string - ? T - : T extends readonly (infer TItem)[] - ? JsonRepresentation[] - : T extends object - ? { [TKey in keyof T]: JsonRepresentation } - : never; - -export interface ReplicatedStateDelivery { - readonly kind: "hydrate" | "update"; - readonly sequence: number; -} - -export interface ReplicatedState { - /** - * Contract-immutable value, or undefined until hydration. It is not frozen and may share containers with an - * in-process provider; consumers must not mutate it. Later updates do not mutate previously returned values. - */ - readonly value: T | undefined; - /** - * Each subscription serializes callbacks, awaiting hydration before updates. Values are immutable and may - * share unchanged revision data. At most 100 deliveries wait behind the running callback; overflow keeps - * only the newest pending value/context/delivery, so update sequences may skip. Failures are reported in - * isolation and delivery continues. Unsubscribe discards pending work without aborting or joining a callback. - */ - subscribe(listener: (value: T, context: Context, delivery: ReplicatedStateDelivery) => Promise): () => void; - subscribe(listener: (value: T, context: Context, delivery: ReplicatedStateDelivery) => void): () => void; -} - -export interface MutableReplicatedState extends ReplicatedState { - readonly value: T; - /** - * Atomically publish one synchronous overlay mutation. Draft handles are unusable after the callback returns. - * Values placed through the draft are cloned by value; assigning `undefined` to an object property deletes it. - */ - change(context: Context, mutate: (draft: Draft) => void): void; - /** - * Atomically take immutable ownership of an alias-free strict-JSON replacement. - * The caller must not mutate the transferred root after this call. - */ - replace(context: Context, value: T): void; -} - -/** One immutable authoritative revision committed after an attachment snapshot. */ -export interface ReplicatedStateSourceFrame { - /** Monotonic source cursor. The first frame after a snapshot must be `snapshot.cursor + 1`. */ - readonly cursor: number; - /** The exact immutable value produced by this commit. */ - readonly value: T; - /** The exact immutable operation batch that produced `value` from the preceding source revision. */ - readonly ops: readonly Op[]; - readonly context: Context; -} - -export interface ReplicatedStateSourceAttachment { - /** Fixed immutable snapshot captured at the atomic attachment boundary. */ - readonly snapshot: { readonly value: T; readonly cursor: number }; - /** - * Install the sole listener and synchronously drain every buffered frame in source commit order. - * This method is single-use. After it begins, every new committed frame must also be delivered in order - * until disposal, including commits made reentrantly while a prior frame is being delivered. - */ - activate(listener: (frame: ReplicatedStateSourceFrame) => void): void; - /** Stop delivery and release source resources. Disposal must be idempotent. */ - dispose(): void; +export interface GetOptions { + /** Principal the record is created for. Defaults to the host principal. */ + readonly principal?: Principal; + /** Host name. Only the local host in phase 1a. */ + readonly at?: string; } /** - * An authoritative immutable revision source. - * - * `attach()` must synchronously and atomically capture one snapshot and register the returned attachment to buffer - * every later committed frame. The snapshot must include every commit before that boundary; buffered frames must - * include every commit after it, with no overlap or gap. Snapshot values, frame values, and operation batches are - * immutable and remain valid after delivery. Chord only publishes these references; it never applies or re-diffs them. + * The explicit call context of one root (a short Chord call tree). Calls made with it carry its root id and epoch; + * `get` holds are released when the call that received this ctx settles. */ -export interface ReplicatedStateSource { - attach(): ReplicatedStateSourceAttachment; +export interface Ctx { + readonly signal: AbortSignal; + readonly rootId: string; + readonly epoch: number; + readonly principal: Principal; + get(type: C, input: InputOf, options?: GetOptions): Handle; } -export interface ReplicatedStateSourceOptions { - /** Receives source-contract and publication-listener failures without throwing them into the source. */ - readonly onError?: (error: Error) => void; -} +export type StateEvent = + | { readonly type: "snap"; readonly value: T } + | { readonly type: "ops"; readonly ops: readonly Op[]; readonly value: T } + | { readonly type: "reset"; readonly value: T }; -/** A synchronously hydrated publication-only state backed by one source attachment. */ -export interface AttachedReplicatedState extends ReplicatedState { +/** A readable state source: a cell, a derivation, or a handle's state member. Values are immutable by contract. */ +export interface Readable { readonly value: T; - /** Idempotently release the source attachment. The last published value remains readable. */ - dispose(): void; -} - -declare const SERVICE_TYPE: unique symbol; - -export type ServiceMode = "singleton" | "keyed"; - -/** Stable identity for one shared TypeScript service contract. */ -export interface Service { - readonly id: string; - /** Process-local services accept unrestricted object contracts and are never published remotely. */ - readonly local: boolean; - readonly [SERVICE_TYPE]?: (value: T) => T; -} - -/** JSON with readonly containers, such as other packages' recursive JSON types. */ -type ReadonlyJsonValue = - | null - | boolean - | number - | string - | readonly ReadonlyJsonValue[] - | { readonly [key: string]: ReadonlyJsonValue }; - -// A type assignable to readonly JSON is valid as is. Walking a recursive JSON type instead, for example one with -// readonly arrays, would never terminate. -type InvalidJsonPart = IsAny extends true - ? T - : unknown extends T - ? never - : [T] extends [ReadonlyJsonValue] - ? never - : InvalidJsonStructure; - -type InvalidJsonProperty = [Exclude] extends [never] ? T : InvalidJsonPart>; - -type InvalidJsonStructure = T extends null | boolean | number | string - ? never - : T extends readonly (infer TItem)[] - ? InvalidJsonPart - : T extends (...args: never[]) => unknown - ? T - : T extends object - ? { [TKey in keyof T]-?: InvalidJsonProperty }[keyof T] - : T; - -type InvalidRemoteMember = T extends ReplicatedState - ? InvalidJsonPart extends never - ? never - : "state value is not JSON" - : T extends (...args: [...infer TArgs, Context]) => Promise - ? InvalidJsonPart extends never - ? TResult extends void - ? never - : InvalidJsonPart extends never - ? never - : "method result is not JSON or void" - : "method argument is not JSON" - : "member is not a remote method or ReplicatedState"; - -type InvalidRemoteMemberNames = { - [TKey in keyof T]-?: InvalidRemoteMember extends never ? never : TKey; -}[keyof T]; - -export type RemoteServiceContract = InvalidRemoteMemberNames extends never ? T : never; - -export interface ServiceSpawner { - spawn(key: string, implementation: T): () => void; -} - -export interface RemoteServices { - use(service: Service): T; - observe(service: Service, handler: (service: T, context: Context) => void | Promise): () => void; - /** Wait until every currently acquired service has installed its initial snapshot. */ - ready(context: Context): Promise; - dispose(context: Context): Promise; -} - -export type ServiceCatalogueEntry = { - readonly serviceId: string; - readonly mode: ServiceMode; -}; - -export type ServiceInstanceAddress = { - readonly key: string; - readonly generation: number; -}; - -export type ServiceMemberSnapshot = - | { readonly name: string; readonly kind: "method" } - | { readonly name: string; readonly kind: "state"; readonly sequence: number; readonly ops: readonly Op[] }; - -export type ServiceInstanceSnapshot = { - readonly instance?: ServiceInstanceAddress; - readonly members: readonly ServiceMemberSnapshot[]; -}; - -export type ServiceSubscriptionSnapshot = { - readonly serviceId: string; - readonly mode: ServiceMode; - readonly instances: readonly ServiceInstanceSnapshot[]; -}; - -export type ServiceProviderUpdate = - | { - readonly type: "state"; - readonly instance?: ServiceInstanceAddress; - readonly member: string; - readonly sequence: number; - readonly ops: readonly Op[]; - } - /** Full subscription rebaseline after overflow; every state is a root replacement at its new sequence. */ - | { readonly type: "reset"; readonly snapshot: ServiceSubscriptionSnapshot } - | { readonly type: "unavailable" } - | { readonly type: "replaced"; readonly snapshot: ServiceInstanceSnapshot } - | { readonly type: "spawned"; readonly instance: ServiceInstanceSnapshot } - | { readonly type: "closed"; readonly instance: ServiceInstanceAddress }; - -export type ServiceCall = { - readonly serviceId: string; - readonly instance?: ServiceInstanceAddress; - readonly member: string; - /** Borrowed immutable values. Chord validates but does not clone them. */ - readonly args: readonly JsonValue[]; -}; - -export interface ServiceSubscription { - /** Atomic baseline. Later updates buffer until activation, with a full reset on pending delivery 101. */ - readonly snapshot: ServiceSubscriptionSnapshot; - activate(): void; - close(context?: Context): void | Promise; + /** Called with the current value immediately, then with every new value. */ + subscribe(listener: (value: T) => void): () => void; + /** Delta stream: a snapshot immediately, then ops, or a reset after a migration or rebind. */ + events(listener: (event: StateEvent) => void): () => void; } /** - * Pluggable wire boundary consumed by a remote service binding. - * - * Implementations choose transport, framing, routing, and envelope encoding. Values crossing this - * boundary must remain strict JSON. Chord does not clone values or require a particular application wire protocol; - * adapters own serialization and any isolation copies they require. - * + * What a behavior may expose as a state member: a cell, a derivation, or a held object's state member (whose value is + * `undefined` only while that object is loading; a record is live only after its holds are). */ -export interface RemoteServiceTransport { - invoke(call: ServiceCall, context: Context): Promise; - subscribe( - serviceId: string, - mode: ServiceMode, - listener: (update: ServiceProviderUpdate, context: Context) => void, - context: Context, - ): Promise; +export type StateSource = Readable | Readable; + +declare const EXPOSED: unique symbol; + +/** A local member backed by a resource (`self.expose(res.x)`): handles read the resource's value once live. */ +export interface Exposed { + readonly [EXPOSED]: T; } -export interface RemoteServiceBindingOptions { - readonly services: readonly { readonly id: string }[]; - readonly transport: RemoteServiceTransport; - readonly bound?: boolean; - readonly onError?: (error: Error) => void; - readonly assertAccess?: () => void; +export type HandleStatus = "loading" | "live" | "released" | "gone"; + +export interface Shape { + readonly kind: string | undefined; + readonly members: readonly string[]; + /** Fingerprint of the provider contract. */ + readonly contract: string; + readonly epoch: number; } -export interface RemoteServiceBinding extends RemoteServices { - rebind(bound: boolean, context: Context): Promise; -} - -export interface FacetEnvironment { - /** Declare a hard dependency on one singleton service and return its stable handle. */ - use(service: Service): T; - /** Declare a hard dependency on a keyed service and observe each live instance. */ - observe(service: Service, handler: (service: T, context: Context) => void | Promise): void; - /** Declare and install this facet's singleton implementation of a service. */ - provide(service: Service, implementation: NoInfer): void; - /** Declare ownership of a multi-instance service and return its deferred spawning capability. */ - provideMany(service: Service): ServiceSpawner; +export interface HandleBase { + /** The consumer contract token this handle was bound with. */ + readonly contract: C; + readonly status: HandleStatus; + readonly ready: Promise; + readonly kind: string | undefined; + readonly shape: Shape | undefined; + /** Why the handle is gone, if it is. */ + readonly error: unknown; + onShape(listener: (shape: Shape) => void): () => void; /** - * Create initialized mutable state by taking immutable ownership of an alias-free strict-JSON root. - * The caller must not mutate `initial` after this call. + * Called after every status change (`loading` → `live`, `live` → `loading` while a link reconnects, `released`, + * `gone` with `error`). A UI watches for `gone` to `get` again. */ - replicatedState(initial: T): MutableReplicatedState; - /** Give the facet ownership of a resource cleanup function. */ - own(disposal: () => void | Promise): void; - /** Register asynchronous initialization after dependencies are bound and ready. */ - onActivate(callback: () => void | Promise): void; - /** Register final facet teardown. */ - onDeactivate(callback: () => void | Promise): void; -} - -export interface Facet { - readonly id: string; - setup(env: FacetEnvironment): void; -} - -export interface RemoteServiceSource { - /** Whether this currently unavailable source may provisionally own absent requirements. */ - readonly acceptsUnavailableServices: boolean; - catalogue(context: Context): Promise; - open(options: { - readonly services: readonly { readonly id: string }[]; - assertAccess(): void; - onError(error: Error): void; - }): RemoteServices; -} - -export interface FacetOptions { - readonly facets: readonly Facet[]; - readonly serviceSources?: readonly RemoteServiceSource[]; - readonly onError?: (error: Error) => void; -} - -export interface FacetHost { - readonly services: RemoteServiceProvider; - /** Activate and replace facets with matching IDs without disconnecting consumer service handles. */ - reload(facets: readonly Facet[]): Promise; - dispose(): Promise; -} - -export interface LoadedFacets { - readonly facets: readonly Facet[]; - dispose(): Promise; -} - -export interface FacetLoader { - load(): Promise; + onStatus(listener: (status: HandleStatus, error: unknown) => void): () => void; + release(): void; } diff --git a/packages/chord/src/websocket.ts b/packages/chord/src/websocket.ts new file mode 100644 index 000000000..a5edc17a5 --- /dev/null +++ b/packages/chord/src/websocket.ts @@ -0,0 +1,34 @@ +import type { AcceptOptions } from "./connection.ts"; +import type { Host } from "./host.ts"; +import { type WebSocketLike, webSocketLink } from "./link.ts"; +import type { Principal } from "./types.ts"; + +/** A WebSocket server (e.g. `ws`'s `WebSocketServer`), structurally: Chord does not depend on `ws`. */ +export interface WebSocketServerLike { + on(event: "connection", listener: (socket: WebSocketLike, request: Request) => void): unknown; +} + +/** WebSocket peers are always accepted as clients: no roots, no epochs, no reload votes, not addressable. */ +export interface WebSocketAcceptOptions extends Omit { + /** The authenticator: the principal of the connecting client, or undefined to refuse it. */ + principal(request: Request): Principal | undefined | Promise; +} + +/** Accepts every connection of `server` on `host` as a client link stamped with the authenticated principal. */ +export function acceptWebSockets( + server: WebSocketServerLike, + host: Host, + options: WebSocketAcceptOptions, +): void { + server.on("connection", (socket, request) => { + // Wrap at once so frames sent while authenticating are queued, not lost. + const link = webSocketLink(socket); + Promise.resolve(options.principal(request)).then( + (principal) => { + if (principal === undefined) link.close("unauthorized"); + else host.accept(link, { ...options, principal, kind: "client" }); + }, + () => link.close("authentication failed"), + ); + }); +} diff --git a/packages/chord/src/wire.ts b/packages/chord/src/wire.ts new file mode 100644 index 000000000..0a90d6872 --- /dev/null +++ b/packages/chord/src/wire.ts @@ -0,0 +1,222 @@ +import { Type } from "typebox"; +import { Value } from "typebox/value"; +import type { Description } from "./contract.ts"; +import type { WireOp } from "./delta/index.ts"; +import { ChordError, type ChordErrorCode } from "./errors.ts"; +import type { Principal } from "./types.ts"; + +// JSON frames between two hosts (or a host and a client). Ids are chosen by the side that creates the hold, call or +// subscription; every frame type has one direction, so ids never collide: +// consumer → provider: hold, release, call, cancel, sub, unsub +// provider → consumer: shape, gone, result, state +// both: hello, ping, pong, bye, root_end; parent → child: invalidate, prepare, commit, abort, retire; child → parent: vote + +export interface RootRef { + readonly rootId: string; + readonly epoch: number; +} + +export interface WireError { + readonly code: ChordErrorCode; + readonly message: string; + readonly reason?: string; +} + +export interface WireReloadOptions { + readonly remove?: readonly string[]; + readonly gateDeadline?: number; + readonly retireDeadline?: number; +} + +/** A child's prepared reload, sent with its vote (same fields as `ReloadReport`, without children). */ +export interface WireReloadReport { + readonly epoch: number; + readonly previous: number; + readonly changed: readonly string[]; + readonly removed: readonly string[]; + readonly prepared: readonly string[]; + readonly gated: readonly string[]; +} + +export type Frame = + | { readonly t: "hello"; readonly name: string; readonly kind: "host" | "client"; readonly epoch: number } + | { + readonly t: "hold"; + readonly id: number; + /** Target host name. */ + readonly at: string; + /** The consumer's contract token; the provider checks that it covers it. */ + readonly token: Description; + readonly input: unknown; + /** A forwarded principal. Honored only on frames from the parent link. */ + readonly principal?: Principal; + /** Set by a relay when the hold originates from a client (browser): not a reload participant. */ + readonly client?: boolean; + } + | { readonly t: "release"; readonly id: number } + | { + readonly t: "shape"; + readonly id: number; + readonly kind?: string; + readonly members: readonly string[]; + readonly contract: string; + readonly epoch: number; + } + | { readonly t: "gone"; readonly id: number; readonly error: WireError } + | { + readonly t: "call"; + readonly id: number; + readonly hold: number; + readonly member: string; + readonly args: unknown; + /** Absent: the receiving host starts a root (browsers never carry roots). */ + readonly root?: RootRef; + } + | { readonly t: "result"; readonly id: number; readonly value?: unknown; readonly error?: WireError } + | { readonly t: "cancel"; readonly id: number } + | { readonly t: "sub"; readonly id: number; readonly hold: number; readonly member: string } + | { readonly t: "unsub"; readonly id: number } + | { + readonly t: "state"; + readonly id: number; + readonly snap?: unknown; + readonly reset?: unknown; + /** Encoded with the subscription's delta encoder. */ + readonly ops?: readonly WireOp[]; + readonly error?: WireError; + } + | { readonly t: "ping"; readonly n: number } + | { readonly t: "pong"; readonly n: number } + | { readonly t: "bye"; readonly reason?: string } + /** The root ended at its origin: its leases on records end (sent on every link the root's calls used). */ + | { readonly t: "root_end"; readonly rootId: string } + | { readonly t: "invalidate"; readonly pattern: Readonly> } + | { + readonly t: "prepare"; + readonly reload: number; + readonly epoch: number; + readonly plugins: readonly string[]; + readonly options: WireReloadOptions; + } + | { readonly t: "vote"; readonly reload: number; readonly error?: WireError; readonly report?: WireReloadReport } + | { readonly t: "commit"; readonly reload: number } + /** The parent dropped `epoch`: roots of it may still come from there no longer. */ + | { readonly t: "retire"; readonly epoch: number } + | { readonly t: "abort"; readonly reload: number }; + +export function toWireError(error: unknown): WireError { + if (error instanceof ChordError) { + return error.reason === undefined + ? { code: error.code, message: error.message } + : { code: error.code, message: error.message, reason: error.reason }; + } + return { code: "unavailable", message: error instanceof Error ? error.message : String(error) }; +} + +export const fromWireError = (error: WireError): ChordError => + new ChordError(error.code, error.message.replace(/^[a-z_]+(\/[a-z_]+)?: /, ""), { reason: error.reason }); + +// ─── validation ────────────────────────────────────────────────────────────── + +const Id = Type.Integer({ minimum: 1 }); +const Name = Type.String({ minLength: 1 }); +const Root = Type.Object({ rootId: Name, epoch: Type.Integer({ minimum: 1 }) }); +const ErrorShape = Type.Object({ code: Type.String(), message: Type.String(), reason: Type.Optional(Type.String()) }); +const Reload = Type.Integer({ minimum: 1 }); + +/** Structural schema of every frame. Payloads (args, values, ops) are checked where they are used. */ +const FrameSchema = Type.Union([ + Type.Object({ + t: Type.Literal("hello"), + name: Name, + kind: Type.Union([Type.Literal("host"), Type.Literal("client")]), + epoch: Type.Integer({ minimum: 0 }), + }), + Type.Object({ + t: Type.Literal("hold"), + id: Id, + at: Name, + token: Type.Object({ + name: Name, + scope: Type.String(), + version: Type.Integer(), + input: Type.Unknown(), + variants: Type.Record(Type.String(), Type.Unknown()), + }), + input: Type.Unknown(), + principal: Type.Optional(Type.Unknown()), + client: Type.Optional(Type.Boolean()), + }), + Type.Object({ t: Type.Literal("release"), id: Id }), + Type.Object({ + t: Type.Literal("shape"), + id: Id, + kind: Type.Optional(Type.String()), + members: Type.Array(Type.String()), + contract: Type.String(), + epoch: Type.Integer(), + }), + Type.Object({ t: Type.Literal("gone"), id: Id, error: ErrorShape }), + Type.Object({ + t: Type.Literal("call"), + id: Id, + hold: Id, + member: Name, + args: Type.Unknown(), + root: Type.Optional(Root), + }), + Type.Object({ + t: Type.Literal("result"), + id: Id, + value: Type.Optional(Type.Unknown()), + error: Type.Optional(ErrorShape), + }), + Type.Object({ t: Type.Literal("cancel"), id: Id }), + Type.Object({ t: Type.Literal("sub"), id: Id, hold: Id, member: Name }), + Type.Object({ t: Type.Literal("unsub"), id: Id }), + Type.Object({ + t: Type.Literal("state"), + id: Id, + snap: Type.Optional(Type.Unknown()), + reset: Type.Optional(Type.Unknown()), + ops: Type.Optional(Type.Array(Type.Array(Type.Unknown()))), + error: Type.Optional(ErrorShape), + }), + Type.Object({ t: Type.Literal("ping"), n: Type.Integer() }), + Type.Object({ t: Type.Literal("pong"), n: Type.Integer() }), + Type.Object({ t: Type.Literal("bye"), reason: Type.Optional(Type.String()) }), + Type.Object({ t: Type.Literal("root_end"), rootId: Name }), + Type.Object({ t: Type.Literal("invalidate"), pattern: Type.Record(Type.String(), Type.Unknown()) }), + Type.Object({ + t: Type.Literal("prepare"), + reload: Reload, + epoch: Type.Integer({ minimum: 1 }), + plugins: Type.Array(Type.String()), + options: Type.Object({ + remove: Type.Optional(Type.Array(Type.String())), + gateDeadline: Type.Optional(Type.Number()), + retireDeadline: Type.Optional(Type.Number()), + }), + }), + Type.Object({ + t: Type.Literal("vote"), + reload: Reload, + error: Type.Optional(ErrorShape), + report: Type.Optional( + Type.Object({ + epoch: Type.Integer(), + previous: Type.Integer(), + changed: Type.Array(Type.String()), + removed: Type.Array(Type.String()), + prepared: Type.Array(Type.String()), + gated: Type.Array(Type.String()), + }), + ), + }), + Type.Object({ t: Type.Literal("commit"), reload: Reload }), + Type.Object({ t: Type.Literal("retire"), epoch: Type.Integer({ minimum: 1 }) }), + Type.Object({ t: Type.Literal("abort"), reload: Reload }), +]); + +/** A frame that passes structural validation; anything else is a protocol violation. */ +export const isFrame = (value: unknown): value is Frame => Value.Check(FrameSchema, value); diff --git a/packages/chord/test/bundle.test.ts b/packages/chord/test/bundle.test.ts deleted file mode 100644 index 626b5df65..000000000 --- a/packages/chord/test/bundle.test.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { afterEach, describe, expect, test } from "vitest"; -import { bundleFacetPackage, bundleFacets } from "../src/bundler.ts"; -import { createFacetHost, defineFacet, defineService } from "../src/index.ts"; -import { - createFacetBundleArtifactLoader, - createFacetBundleLoader, - readFacetBundleArtifact, - readFacetBundleManifest, -} from "../src/node.ts"; - -interface GenerationValue { - read(): string; -} - -const packageDirectory = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const temporaryDirectories: string[] = []; -const GenerationValue = defineService("test.bundle.generation", { local: true }); - -afterEach(async () => { - await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true }))); -}); - -describe("facet bundles", () => { - test("builds independent content-addressed entries and loads fresh reloadable generations", async () => { - const directory = await mkdtemp(join(packageDirectory, ".bundle-test-")); - temporaryDirectories.push(directory); - const sourceDirectory = join(directory, "src"); - const outputDirectory = join(directory, "bundle"); - await mkdir(sourceDirectory); - await writeFile( - join(sourceDirectory, "helper.ts"), - 'export const decorate = (value: string): string => "generation:" + value;\n', - ); - const entryPath = join(sourceDirectory, "entry.ts"); - const presentationPath = join(sourceDirectory, "presentation.ts"); - await writeGeneration(entryPath, "A"); - await writeFile(presentationPath, 'export default { id: "bundle-presentation", setup() {} };\n'); - const facetEntries = { presentation: presentationPath, worker: entryPath }; - - const firstBuild = await bundleFacets({ - plugin: { id: "test-bundle", version: "1" }, - entries: facetEntries, - outdir: outputDirectory, - sourceMap: true, - }); - const firstEntry = firstBuild.manifest.entries.worker!; - expect(firstEntry.file).toMatch(/^facet-[a-f0-9]{12}-[A-Z0-9]+\.cjs$/u); - expect(firstEntry.sourceMap).toBe(`${firstEntry.file}.map`); - expect(firstEntry.externalImports).toEqual(["@earendil-works/chord"]); - expect(firstBuild.manifest.entries.presentation!.file).not.toBe(firstEntry.file); - expect((await readdir(outputDirectory)).filter((path) => path.endsWith(".cjs"))).toHaveLength(2); - const firstSource = await readFile(join(outputDirectory, firstEntry.file), "utf8"); - expect(firstSource).toContain('require("@earendil-works/chord")'); - expect(firstSource).toContain("module.exports"); - expect((await readFile(firstBuild.manifestPath, "utf8")).endsWith("\n")).toBe(true); - - const presentation = await createFacetBundleLoader({ - manifestPath: firstBuild.manifestPath, - entry: "presentation", - }).load(); - expect(presentation.facets.map(({ id }) => id)).toEqual(["bundle-presentation"]); - await presentation.dispose(); - - const artifact = await readFacetBundleArtifact({ - manifestPath: firstBuild.manifestPath, - entry: "worker", - }); - const materializedDirectory = join(directory, "materialized"); - const transported = await createFacetBundleArtifactLoader({ - artifact: structuredClone(artifact), - temporaryDirectory: materializedDirectory, - }).load(); - expect(transported.facets.map(({ id }) => id)).toEqual(["bundle-provider"]); - expect(await readdir(materializedDirectory)).toHaveLength(1); - await transported.dispose(); - expect(await readdir(materializedDirectory)).toEqual([]); - - const secondBuild = await bundleFacets({ - plugin: { id: "test-bundle", version: "1" }, - entries: facetEntries, - outdir: outputDirectory, - sourceMap: true, - }); - expect(secondBuild.manifest.entries.worker).toEqual(firstEntry); - - const loader = createFacetBundleLoader({ manifestPath: secondBuild.manifestPath, entry: "worker" }); - const loadedA = await loader.load(); - const loadedACopy = await loader.load(); - expect(loadedACopy.facets[0]).not.toBe(loadedA.facets[0]); - await loadedACopy.dispose(); - - let retained: GenerationValue | undefined; - const consumer = defineFacet({ - id: "bundle-consumer", - setup(env) { - retained = env.use(GenerationValue); - }, - }); - const host = await createFacetHost({ facets: [consumer, ...loadedA.facets] }); - expect(retained!.read()).toBe("generation:A"); - - await writeGeneration(entryPath, "B"); - const thirdBuild = await bundleFacets({ - plugin: { id: "test-bundle", version: "2" }, - entries: facetEntries, - outdir: outputDirectory, - sourceMap: true, - }); - expect(thirdBuild.manifest.entries.worker!.file).not.toBe(firstEntry.file); - const loadedB = await loader.load(); - await host.reload(loadedB.facets); - await loadedA.dispose(); - expect(retained!.read()).toBe("generation:B"); - - await host.dispose(); - await loadedB.dispose(); - }); - - test("loads host externals through the controlled CommonJS require", async () => { - const directory = await mkdtemp(join(packageDirectory, ".bundle-external-test-")); - temporaryDirectories.push(directory); - const externalPath = join(directory, "host.mjs"); - const entryPath = join(directory, "entry.ts"); - const outputDirectory = join(directory, "bundle"); - await writeFile(externalPath, 'export const named = "host-named";\n'); - await writeFile( - entryPath, - 'import { named } from "@example/host";\n' + - 'export const loadDynamic = () => import("@example/dynamic");\n' + - 'if (named !== "host-named") throw new Error("external mismatch");\n' + - 'export default { id: "external-facet", setup() {} };\n', - ); - const result = await bundleFacets({ - plugin: { id: "external-bundle" }, - entries: { worker: entryPath }, - external: ["@example/host", "@example/dynamic"], - outdir: outputDirectory, - }); - const entry = result.manifest.entries.worker!; - const source = await readFile(join(outputDirectory, entry.file), "utf8"); - expect(source).not.toContain("import("); - expect(source).toContain('require("@example/dynamic")'); - const loaded = await createFacetBundleLoader({ - manifestPath: result.manifestPath, - entry: "worker", - resolveExternal: (specifier) => - specifier === "@example/host" || specifier === "@example/dynamic" ? pathToFileURL(externalPath) : undefined, - }).load(); - expect(loaded.facets.map(({ id }) => id)).toEqual(["external-facet"]); - await loaded.dispose(); - }); - - test("builds plugin packages from conventional and configured facet entries", async () => { - const directory = await mkdtemp(join(packageDirectory, ".bundle-package-test-")); - temporaryDirectories.push(directory); - const sourceDirectory = join(directory, "src"); - await mkdir(sourceDirectory); - await Promise.all([ - writeFile( - join(directory, "package.json"), - `${JSON.stringify({ - name: "@example/conventional-plugin", - version: "1.2.3", - peerDependencies: { "@example/host": "^1.0.0" }, - })}\n`, - ), - writeFile( - join(sourceDirectory, "session.ts"), - 'import "@example/host/plugin"; export default { id: "package-session", setup() {} };\n', - ), - writeFile(join(sourceDirectory, "tui.ts"), 'export default { id: "package-tui", setup() {} };\n'), - writeFile(join(sourceDirectory, "contract.ts"), "export const ignored = true;\n"), - writeFile(join(sourceDirectory, "presentation.ts"), 'export default { id: "configured-tui", setup() {} };\n'), - ]); - - const conventional = await bundleFacetPackage({ - packagePath: directory, - outdir: join(directory, "build"), - defaultFacets: { session: "src/session.ts", tui: "src/tui.ts", browser: "src/browser.ts" }, - }); - expect(conventional.packageDirectory).toBe(directory); - expect(conventional.manifest.plugin).toEqual({ id: "@example/conventional-plugin", version: "1.2.3" }); - expect(Object.keys(conventional.manifest.entries)).toEqual(["session", "tui"]); - expect(conventional.manifest.entries.session?.externalImports).toEqual(["@example/host/plugin"]); - expect(conventional.manifest.entries.tui?.sourceMap).toMatch(/\.cjs\.map$/u); - - await writeFile( - join(directory, "package.json"), - `${JSON.stringify({ - name: "@example/conventional-plugin", - version: "2.0.0", - chord: { facets: { session: false, tui: "src/presentation.ts" }, sourceMap: false }, - })}\n`, - ); - const configured = await bundleFacetPackage({ - packagePath: join(directory, "package.json"), - outdir: join(directory, "build"), - defaultFacets: { session: "src/session.ts", tui: "src/tui.ts" }, - }); - expect(Object.keys(configured.manifest.entries)).toEqual(["tui"]); - expect(configured.manifest.entries.tui?.sourceMap).toBeUndefined(); - const loaded = await createFacetBundleLoader({ - manifestPath: configured.manifestPath, - entry: "tui", - }).load(); - expect(loaded.facets.map(({ id }) => id)).toEqual(["configured-tui"]); - await loaded.dispose(); - }); - - test("rejects invalid plugin package entry configuration", async () => { - const directory = await mkdtemp(join(packageDirectory, ".bundle-package-test-")); - temporaryDirectories.push(directory); - await writeFile( - join(directory, "package.json"), - `${JSON.stringify({ - name: "invalid-plugin", - version: "1.0.0", - chord: { facets: { tui: "../outside.ts" } }, - })}\n`, - ); - await expect(bundleFacetPackage({ packagePath: directory, outdir: join(directory, "build") })).rejects.toThrow( - "escapes the package directory", - ); - }); - - test("rejects corrupt entries and invalid module exports", async () => { - const directory = await mkdtemp(join(packageDirectory, ".bundle-test-")); - temporaryDirectories.push(directory); - const entryPath = join(directory, "entry.ts"); - const outputDirectory = join(directory, "bundle"); - await writeFile(entryPath, "export default { id: 'missing-setup' };\n"); - const result = await bundleFacets({ - plugin: { id: "invalid-bundle" }, - entries: { invalid: entryPath }, - outdir: outputDirectory, - }); - const loader = createFacetBundleLoader({ manifestPath: result.manifestPath, entry: "invalid" }); - await expect(loader.load()).rejects.toThrow("has no setup function"); - - const manifest = await readFacetBundleManifest(result.manifestPath); - await writeFile(join(outputDirectory, manifest.entries.invalid!.file), "export default {};\n"); - await expect(loader.load()).rejects.toThrow("integrity check failed"); - }); -}); - -async function writeGeneration(path: string, generation: string): Promise { - await writeFile( - path, - `import "@earendil-works/chord";\n` + - `import { decorate } from "./helper.ts";\n` + - `const Value = { id: "test.bundle.generation", local: true };\n` + - `export default { id: "bundle-provider", setup(env) {\n` + - ` env.provide(Value, { read() { return decorate(${JSON.stringify(generation)}); } });\n` + - `}};\n`, - ); -} diff --git a/packages/chord/test/cluster.test.ts b/packages/chord/test/cluster.test.ts new file mode 100644 index 000000000..7d46ce1f7 --- /dev/null +++ b/packages/chord/test/cluster.test.ts @@ -0,0 +1,278 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it, onTestFinished } from "vitest"; +import { + type Ctx, + cell, + createHost, + type Facet, + type FacetLoader, + isChordError, + memoryLinkPair, + method, + remote, + resource, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); + +function latch() { + let open = () => {}; + const promise = new Promise((resolve) => { + open = resolve; + }); + return { promise, open }; +} + +const Greeter = remote("t.greeter", { + version: 1, + input: Type.Object({}), + variants: { + main: { + state: { count: Type.Object({ n: Type.Number() }) }, + methods: { greet: method(Type.String(), Type.String()) }, + }, + }, +}); + +const GreeterV2 = remote("t.greeter", { + version: 2, + input: Type.Object({}), + variants: { main: { methods: { hello: method(Type.String(), Type.String()) } } }, +}); + +function greeterFacet(greeting: string, options: { fail?: boolean } = {}): Facet { + return { + name: "greeter", + setup: (api) => + api.serve(Greeter, (_input, _principal, self) => { + if (options.fail) throw new Error("worker factory fails"); + const state = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + return { + kind: "main", + count: state.count, + greet: (name: string, ctx: Ctx) => { + state.count.change(ctx, (draft) => { + draft.n += 1; + }); + return `${greeting} ${name}`; + }, + }; + }), + }; +} + +/** Server + worker over a memory link; the worker loads facets by name from `modules`. */ +function setup() { + const modules = new Map([["greeter", greeterFacet("v1")]]); + const loader: FacetLoader = async (names) => names.map((name) => modules.get(String(name))!); + const server = createHost({ name: "server", onError: () => {} }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "w" }, kind: "host", name: "session:S1" }); + const worker = createHost({ name: "session:S1", upstream: up, loader, onError: () => {} }); + worker.install(modules.get("greeter")!); + return { server, worker, modules, down }; +} + +describe("cross-host reload", () => { + it("prepares every host, commits the server first, then the workers; epochs stay in step", async () => { + const { server, worker, modules } = setup(); + await tick(); + const greeter = server.get(Greeter, {}, { at: "session:S1" }); + assert.equal(await greeter.greet("a"), "v1 a"); + const go = latch(); + const pinned = server.root(async (ctx) => { + await greeter.greet("p", ctx); + await go.promise; + return greeter.greet("p", ctx); + }); + await tick(); + const shapes: number[] = []; + greeter.onShape((shape) => shapes.push(shape.epoch)); + modules.set("greeter", greeterFacet("v2")); + const report = await server.reload([], { remote: { "session:": ["greeter"] } }); + // No waiting: the commit frame precedes this call on the FIFO link, so the worker has committed when it arrives. + assert.equal(await greeter.greet("b"), "v2 b"); + assert.equal(server.epoch, report.epoch); + assert.equal(worker.epoch, report.epoch); + go.open(); + assert.equal(await pinned, "v1 p"); + assert.deepEqual(shapes, [report.epoch]); + // The server retires the old epoch once its last old root ended, then tells the worker. + for (let i = 0; i < 20 && worker.previousEpoch !== undefined; i++) await tick(); + assert.equal(server.previousEpoch, undefined); + assert.equal(worker.previousEpoch, undefined); + }); + + it("a failing vote aborts every host; the error keeps its code and names the host", async () => { + const { server, worker, modules } = setup(); + await tick(); + const greeter = server.get(Greeter, {}, { at: "session:S1" }); + await greeter.greet("a"); + const before = [server.epoch, worker.epoch]; + modules.set("greeter", greeterFacet("v2", { fail: true })); + await assert.rejects(server.reload([], { remote: { "session:": ["greeter"] } }), (error) => { + assert.ok(isChordError(error, "unavailable")); + assert.match(error.message, /session:S1: .*worker factory fails/); + return true; + }); + assert.deepEqual([server.epoch, worker.epoch], before); + assert.equal(await greeter.greet("x"), "v1 x"); + }); + + it("a worker gate blocked past its deadline aborts the whole reload", async () => { + const { server, worker, modules } = setup(); + await tick(); + const greeter = server.get(Greeter, {}, { at: "session:S1" }); + const go = latch(); + const blocker = server.root(async (ctx) => { + await greeter.greet("g", ctx); + await go.promise; + }); + await tick(); + const before = [server.epoch, worker.epoch]; + const bumped: Facet = { + name: "greeter", + setup: (api) => + greeterFacet("v2").setup({ + ...api, + serve: ((type: never, factory: never) => api.serve(type, factory, { state: { version: 2 } })) as never, + }), + }; + modules.set("greeter", bumped); + await assert.rejects(server.reload([], { remote: { "session:": ["greeter"] }, gateDeadline: 30 }), (error) => { + assert.ok(isChordError(error, "aborted") && error.reason === "gate_deadline"); + assert.match(error.message, /session:S1: .*blocking roots: server#\w+:\d+/); + return true; + }); + assert.deepEqual([server.epoch, worker.epoch], before); + go.open(); + await blocker; + assert.equal(await greeter.greet("x"), "v1 x"); + }); + + it("removes facets from server and workers, reconciles removal tombstones, and clears them on re-add", async () => { + const { server, worker, modules } = setup(); + onTestFinished(async () => { + await worker.close(); + await server.close(); + }); + await server.install(greeterFacet("v1")); + await tick(); + const local = server.get(Greeter, {}); + const child = server.get(Greeter, {}, { at: "session:S1" }); + assert.equal(await local.greet("a"), "v1 a"); + assert.equal(await child.greet("a"), "v1 a"); + await server.reload([], { remove: ["greeter"], remoteRemove: { "session:": ["greeter"] } }); + for (let i = 0; i < 20 && child.status !== "gone"; i++) await tick(); + assert.equal(local.status, "gone"); + assert.equal(child.status, "gone"); + + // A child starting with the old set must remove it without trying to import its removed module. + const cold = createHost({ + name: "session:cold", + loader: async () => { + assert.fail("Removed modules must not be imported"); + }, + onError: () => {}, + }); + onTestFinished(() => cold.close()); + await cold.install(greeterFacet("v1")); + const old = cold.get(Greeter, {}); + await old.ready; + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "cold" }, kind: "host", name: "session:cold" }); + cold.connect(up); + for (let i = 0; i < 20 && old.status !== "gone"; i++) await tick(); + assert.equal(old.status, "gone"); + assert.equal(cold.epoch, server.epoch); + // Close this loader-less child before the addition; it tested only the removal reconciliation. + await cold.close(); + modules.set("greeter", greeterFacet("v2")); + await server.reload([greeterFacet("v2")], { remote: { "session:": ["greeter"] } }); + assert.equal(await server.get(Greeter, {}, { at: "session:S1" }).greet("b"), "v2 b"); + const fresh = createHost({ + name: "session:fresh", + loader: async (names) => names.map((name) => modules.get(String(name))!), + onError: () => {}, + }); + onTestFinished(() => fresh.close()); + await fresh.install(greeterFacet("v1")); + const [freshDown, freshUp] = memoryLinkPair(); + server.accept(freshDown, { principal: { worker: "fresh" }, kind: "host", name: "session:fresh" }); + fresh.connect(freshUp); + assert.equal(await server.get(Greeter, {}, { at: "session:fresh" }).greet("c"), "v2 c"); + }); + + it("only the server coordinates reloads", async () => { + const { worker } = setup(); + await tick(); + await assert.rejects(worker.reload([]), (error) => isChordError(error, "unavailable")); + }); + + it("browsers are not participants: they see shape and reset pushes, or upgrade_required", async () => { + const { server, modules, down: workerLink } = setup(); + // The route resolves to the already connected worker. + server.route("session:", { authorize: () => true, resolve: async () => workerLink }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + const viewer = browser.get(Greeter, {}, { at: "session:S1" }); + assert.equal(await viewer.greet("a"), "v1 a"); + const shapes: number[] = []; + viewer.onShape((shape) => shapes.push(shape.epoch)); + modules.set("greeter", greeterFacet("v2")); + const report = await server.reload([], { remote: { "session:": ["greeter"] } }); + for (let i = 0; i < 20 && shapes.length === 0; i++) await tick(); + assert.deepEqual(shapes, [report.epoch]); + assert.equal(await viewer.greet("b"), "v2 b"); + + modules.set("greeter", { + name: "greeter", + setup: (api) => api.serve(GreeterV2, () => ({ kind: "main", hello: (name: string) => `hi ${name}` })), + }); + await server.reload([], { remote: { "session:": ["greeter"] } }); + for (let i = 0; i < 20 && viewer.status !== "gone"; i++) await tick(); + assert.equal(viewer.status, "gone"); + assert.ok(isChordError(viewer.error, "upgrade_required")); + }); + + it("an abort while a worker is still preparing: a late open closes itself and the worker can reload again", async () => { + const { server, worker, modules } = setup(); + await tick(); + const greeter = server.get(Greeter, {}, { at: "session:S1" }); + await greeter.greet("a"); + const log: string[] = []; + const go = latch(); + const slow: Facet = { + name: "greeter", + setup: (api) => + api.serve(Greeter, (_input, _principal, self) => { + self.resources({ + late: resource({ + rev: 1, + open: async () => { + await go.promise; + log.push("opened"); + }, + close: () => void log.push("closed"), + }), + }); + const state = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + return { kind: "main", count: state.count, greet: (name: string) => `slow ${name}` }; + }), + }; + modules.set("greeter", slow); + const epoch = worker.epoch; + await assert.rejects(server.reload([], { remote: { "session:": ["greeter"] }, voteDeadline: 50 }), (error) => + isChordError(error, "aborted"), + ); + go.open(); + for (let i = 0; i < 50 && !log.includes("closed"); i++) await tick(); + assert.deepEqual(log, ["opened", "closed"]); + assert.equal(worker.epoch, epoch); + modules.set("greeter", greeterFacet("v3")); + await server.reload([], { remote: { "session:": ["greeter"] } }); + assert.equal(await greeter.greet("b"), "v3 b"); + }); +}); diff --git a/packages/chord/test/contract.test.ts b/packages/chord/test/contract.test.ts new file mode 100644 index 000000000..e22172984 --- /dev/null +++ b/packages/chord/test/contract.test.ts @@ -0,0 +1,124 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { bind, coverage, isChordError, local, method, opaque, remote } from "../src/index.ts"; +import { canonicalIdentity, fits, normalize } from "../src/schema.ts"; + +const View = Type.Object({ text: Type.String() }); + +const chatV1 = remote("t.chat", { + version: 1, + input: Type.Object({ session: Type.String(), conversation: Type.Number({ default: 0 }) }), + variants: { + viewer: { state: { view: View } }, + participant: { + state: { view: View }, + methods: { send: method(Type.Object({ text: Type.String() }), Type.Object({ id: Type.Number() })) }, + }, + }, +}); + +describe("contracts", () => { + it("normalizes input and omits defaults from the identity", () => { + const a = normalize(chatV1.input, { session: "s", extra: 1 }, "input"); + assert.deepEqual(a, { session: "s", conversation: 0 }); + const b = normalize(chatV1.input, { conversation: 0, session: "s" }, "input"); + assert.equal(canonicalIdentity(chatV1.input, a), canonicalIdentity(chatV1.input, b)); + assert.equal(canonicalIdentity(chatV1.input, a), '{"session":"s"}'); + assert.throws( + () => normalize(chatV1.input, { session: 1 }, "input"), + (error) => isChordError(error, "invalid") && error.reason === "input", + ); + }); + + it("fingerprints are stable and description-based", () => { + const copy = remote("t.chat", chatV1.def); + assert.equal(copy.fingerprint, chatV1.fingerprint); + assert.notEqual(remote("t.chat", { ...chatV1.def, version: 2 }).fingerprint, chatV1.fingerprint); + }); + + it("rejects reserved member names and opaque remote members", () => { + assert.throws(() => + local("t.x", { version: 1, input: Type.Object({}), members: { status: method(Type.Null(), Type.Null()) } }), + ); + assert.throws(() => + remote("t.x", { + version: 1, + input: Type.Object({}), + variants: { a: { state: { h: opaque("x@1") as never } } }, + }), + ); + }); + + it("covers additive changes and refuses breaking ones", () => { + const additive = remote("t.chat", { + ...chatV1.def, + variants: { + ...chatV1.def.variants, + participant: { + state: { view: Type.Object({ text: Type.String(), extra: Type.Number() }) }, + methods: { + send: method( + Type.Object({ text: Type.String(), urgent: Type.Optional(Type.Boolean()) }), + Type.Object({ id: Type.Number(), at: Type.Number() }), + ), + }, + }, + }, + }); + assert.equal(coverage(additive.description, chatV1.description), undefined); + assert.equal(bind(additive, undefined, chatV1), undefined); + + const required = remote("t.chat", { + ...chatV1.def, + variants: { + ...chatV1.def.variants, + participant: { + state: { view: View }, + methods: { + send: method( + Type.Object({ text: Type.String(), urgent: Type.Boolean() }), + Type.Object({ id: Type.Number() }), + ), + }, + }, + }, + }); + assert.match(coverage(required.description, chatV1.description) ?? "", /arguments/); + assert.throws( + () => bind(required, undefined, chatV1), + (error) => isChordError(error, "contract_mismatch"), + ); + + const retyped = remote("t.chat", { + ...chatV1.def, + variants: { ...chatV1.def.variants, viewer: { state: { view: Type.Object({ text: Type.Number() }) } } }, + }); + assert.match(coverage(retyped.description, chatV1.description) ?? "", /state/); + }); + + it("binds other versions only through accepts adapters", () => { + const v2 = remote("t.chat", { ...chatV1.def, version: 2, variants: { viewer: { state: { body: View } } } }); + assert.throws( + () => bind(v2, undefined, chatV1), + (error) => isChordError(error, "contract_mismatch"), + ); + const adapter = (current: Record) => ({ kind: "viewer", view: current.body }); + assert.equal(bind(v2, { 1: adapter }, chatV1), adapter); + }); + + it("compares opaque members by name@version and function members structurally", () => { + const a = local("t.idx", { + version: 1, + input: Type.Object({}), + members: { + harness: opaque("h@1"), + onChange: method(Type.Function([Type.String()], Type.Void()), Type.Null()), + }, + }); + const b = local("t.idx", { ...a.def, members: { ...a.def.members, harness: opaque("h@2") } }); + assert.match(coverage(b.description, a.description) ?? "", /opaque/); + assert.ok(fits(Type.Function([Type.Unknown()], Type.Void()), Type.Function([Type.String()], Type.Void()))); + assert.ok(!fits(Type.Function([Type.String()], Type.Void()), Type.Function([Type.Unknown()], Type.Void()))); + }); +}); diff --git a/packages/chord/test/delta-diff.test.ts b/packages/chord/test/delta-diff.test.ts index 495d525a2..e00aa45c7 100644 --- a/packages/chord/test/delta-diff.test.ts +++ b/packages/chord/test/delta-diff.test.ts @@ -8,7 +8,7 @@ import { encoder, type Op, } from "../src/delta/index.ts"; -import type { JsonValue } from "../src/types.ts"; +import type { JsonValue } from "../src/json.ts"; const expectDiff = (before: JsonValue, after: JsonValue, expected: unknown): void => { const operations = diffRevisions(before, after); diff --git a/packages/chord/test/eager-reload.test.ts b/packages/chord/test/eager-reload.test.ts new file mode 100644 index 000000000..991db133b --- /dev/null +++ b/packages/chord/test/eager-reload.test.ts @@ -0,0 +1,306 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { onTestFinished, test } from "vitest"; +import { HostImpl } from "../src/host.ts"; +import { + cell, + defineRegistry, + type Facet, + isChordError, + local, + memoryLinkPair, + method, + resource, + type Self, +} from "../src/index.ts"; + +const input = Type.Object({ key: Type.String() }); +const flags = Type.Record(Type.String(), Type.Boolean()); +const SwitchV1 = local("t.eager-switch", { + version: 1, + input, + members: { + enabled: Type.Boolean(), + set: method(Type.Boolean(), Type.Null()), + }, +}); +const SwitchV2 = local("t.eager-switch", { + version: 2, + input, + members: { + enabled: flags, + set: method(Type.Object({ conversation: Type.Number(), enabled: Type.Boolean() }), Type.Null()), + }, +}); +const SwitchV3 = local("t.eager-switch", { + version: 3, + input, + members: { + enabled: flags, + update: method(Type.Object({ conversation: Type.Number(), enabled: Type.Boolean() }), Type.Null()), + }, +}); +const Read = defineRegistry<() => unknown>("t.eager-switch-read"); + +function switchFacets(log: string[]) { + const declareResource = (self: Self) => + self.resources({ + watch: resource({ + rev: 1, + open: () => { + log.push("open"); + return null; + }, + close: () => { + log.push("close"); + }, + }), + }); + const v1: Facet = { + name: "switch", + setup: (api) => + api.serve( + SwitchV1, + (_input, _principal, self) => { + declareResource(self); + const state = self.state({ enabled: cell(Type.Boolean(), false) }); + self.register(Read, { current: () => state.enabled.value }); + return { + enabled: state.enabled, + set: (enabled, ctx) => { + state.enabled.replace(ctx, enabled); + return null; + }, + }; + }, + { eager: { input: { key: "custom" } } }, + ), + }; + const v2 = (valid: boolean): Facet => ({ + name: "switch", + setup: (api) => + api.serve( + SwitchV2, + (_input, _principal, self) => { + declareResource(self); + const state = self.state({ enabled: cell(flags, {}) }); + self.register(Read, { current: () => state.enabled.value }); + return { + enabled: state.enabled, + set: ({ conversation, enabled }, ctx) => { + state.enabled.change(ctx, (draft) => { + draft[conversation] = enabled; + }); + return null; + }, + }; + }, + { + eager: { input: { key: "custom" } }, + state: { + version: 2, + migrate: { + enabled: (old, from) => { + assert.equal(from, 1); + return valid ? { 1: old as boolean } : "invalid migration"; + }, + }, + }, + }, + ), + }); + const v3: Facet = { + name: "switch", + setup: (api) => + api.serve( + SwitchV3, + (_input, _principal, self) => { + declareResource(self); + const state = self.state({ enabled: cell(flags, {}) }); + self.register(Read, { current: () => state.enabled.value }); + return { + enabled: state.enabled, + update: ({ conversation, enabled }, ctx) => { + state.enabled.change(ctx, (draft) => { + draft[conversation] = enabled; + }); + return null; + }, + }; + }, + { eager: { input: { key: "custom" } }, state: { version: 2 } }, + ), + }; + return { v1, v2, v3 }; +} + +test("an eager hold follows contract bumps at commit without reopening its record or resources; failures retain the old hold", async () => { + const errors: unknown[] = []; + const host = new HostImpl({ dev: true, onError: (error) => errors.push(error) }); + onTestFinished(() => host.close()); + const log: string[] = []; + const facets = switchFacets(log); + await host.load([facets.v1]); + const initial = host.eager.get(SwitchV1.name)!; + const record = initial.record; + const original = host.get(SwitchV1, { key: "custom" }); + await original.set(true); + original.release(); + const epoch = host.epoch; + await assert.rejects(host.reload([facets.v2(false)]), (error) => isChordError(error, "invalid")); + assert.equal(host.epoch, epoch); + assert.equal(host.eager.get(SwitchV1.name), initial); + assert.equal(initial.status, "live"); + assert.equal(host.registry(Read).get("current")!(), true); + const report = await host.reload([facets.v2(true)]); + assert.equal(report.gated.length, 1); + const updated = host.eager.get(SwitchV2.name)!; + assert.equal(updated.contract, SwitchV2); + assert.equal(updated.record, record); + assert.equal(initial.status, "released"); + assert.equal(record?.holds.size, 1, "Attach the replacement before releasing the only old hold"); + assert.deepEqual(host.registry(Read).get("current")!(), { 1: true }); + const modern = host.get(SwitchV2, { key: "custom" }); + await modern.set({ conversation: 2, enabled: true }); + modern.release(); + assert.deepEqual((await host.reload([facets.v3])).gated, []); + assert.equal(host.eager.get(SwitchV3.name)?.contract, SwitchV3); + assert.equal(host.eager.get(SwitchV3.name)?.record, record); + assert.equal(updated.status, "released"); + assert.deepEqual(host.registry(Read).get("current")!(), { 1: true, 2: true }); + await host.ready(); + assert.deepEqual(log, ["open"]); + assert.deepEqual(errors, []); + await host.close(); + assert.deepEqual(log, ["open", "close"]); +}); + +const ReadV1 = local("t.eager-read", { + version: 1, + input: Type.Object({}), + members: { read: method(Type.Null(), Type.String()) }, +}); +const ReadV2 = local("t.eager-read", { + version: 2, + input: Type.Object({}), + members: { fetch: method(Type.Null(), Type.String()) }, +}); +const readerV1: Facet = { + name: "read", + setup: (api) => api.serve(ReadV1, () => ({ read: () => "one" }), { eager: true }), +}; +const readerV2 = (adapt = false): Facet => ({ + name: "read", + setup: (api) => + api.serve(ReadV2, () => ({ fetch: () => "two" }), { + eager: true, + ...(adapt ? { accepts: { 1: (current: Record) => ({ read: current.fetch }) } } : {}), + }), +}); + +test("external old-version consumers still refuse an eager upgrade without accepts, and keep their adapter when it is provided", async () => { + const host = new HostImpl({ + onError: (error) => { + throw error; + }, + }); + onTestFinished(() => host.close()); + await host.load([readerV1]); + const own = host.eager.get(ReadV1.name)!; + const consumer = host.get(ReadV1, {}); + assert.equal(await consumer.read(null), "one"); + await assert.rejects(host.reload([readerV2()]), (error) => isChordError(error, "contract_mismatch")); + assert.equal(host.eager.get(ReadV1.name), own); + assert.equal(await consumer.read(null), "one"); + await host.reload([readerV2(true)]); + assert.equal(host.eager.get(ReadV2.name)?.contract, ReadV2); + assert.equal(await consumer.read(null), "two"); + assert.equal(consumer.contract, ReadV1, "Only the bootstrap hold is retokened"); + assert.equal(await host.get(ReadV2, {}).fetch(null), "two"); +}); + +test("record-owned old-version holds still block an eager upgrade unless their consumer is replaced in the same batch", async () => { + const host = new HostImpl({ onError: () => {} }); + onTestFinished(() => host.close()); + const Consumer = local("t.eager-consumer", { + version: 1, + input: Type.Object({}), + members: { read: method(Type.Null(), Type.String()) }, + }); + const oldConsumer: Facet = { + name: "consumer", + setup: (api) => + api.serve(Consumer, (_input, _principal, self) => { + const reader = self.get(ReadV1, {}); + return { read: (_args, ctx) => reader.read(null, ctx) }; + }), + }; + const newConsumer: Facet = { + name: "consumer", + setup: (api) => + api.serve(Consumer, (_input, _principal, self) => { + const reader = self.get(ReadV2, {}); + return { read: (_args, ctx) => reader.fetch(null, ctx) }; + }), + }; + await host.load([readerV1, oldConsumer]); + const consumer = host.get(Consumer, {}); + assert.equal(await consumer.read(null), "one"); + await assert.rejects(host.reload([readerV2()]), (error) => isChordError(error, "contract_mismatch")); + assert.equal(await consumer.read(null), "one"); + await host.reload([readerV2(), newConsumer]); + assert.equal(await consumer.read(null), "two"); + assert.equal(host.eager.get(ReadV2.name)?.contract, ReadV2); +}); + +test("a forgotten contract bump remains refused even when the only consumer is the host's eager hold", async () => { + const host = new HostImpl({ onError: () => {} }); + onTestFinished(() => host.close()); + await host.load([readerV1]); + const incompatible = local(ReadV1.name, { + version: 1, + input: Type.Object({}), + members: { fetch: method(Type.Null(), Type.String()) }, + }); + await assert.rejects( + host.reload([ + { name: "read", setup: (api) => api.serve(incompatible, () => ({ fetch: () => "bad" }), { eager: true }) }, + ]), + (error) => isChordError(error, "contract_mismatch") && /without a version bump/.test(error.message), + ); + assert.equal(host.eager.get(ReadV1.name)?.contract, ReadV1); +}); + +test("a cluster contract upgrade retokens server and worker eager holds only after unanimous preparation", async () => { + const log: string[] = []; + const facets = switchFacets(log); + let valid = false; + const server = new HostImpl({ onError: () => {} }); + const worker = new HostImpl({ name: "worker", onError: () => {}, loader: async () => [facets.v2(valid)] }); + onTestFinished(async () => { + await worker.close(); + await server.close(); + }); + await server.load([facets.v1]); + await worker.load([facets.v1]); + const [down, up] = memoryLinkPair(); + server.accept(down, { kind: "host", name: "worker", principal: { worker: true } }); + worker.connect(up); + await new Promise((resolve) => setImmediate(resolve)); + const serverOwn = server.eager.get(SwitchV1.name); + const workerOwn = worker.eager.get(SwitchV1.name); + await assert.rejects(server.reload([facets.v2(true)], { remote: { worker: ["switch"] } }), (error) => + isChordError(error, "invalid"), + ); + assert.equal(server.eager.get(SwitchV1.name), serverOwn); + assert.equal(worker.eager.get(SwitchV1.name), workerOwn); + valid = true; + await server.reload([facets.v2(true)], { remote: { worker: ["switch"] } }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(server.eager.get(SwitchV2.name)?.contract, SwitchV2); + assert.equal(worker.eager.get(SwitchV2.name)?.contract, SwitchV2); + assert.deepEqual(server.registry(Read).get("current")!(), { 1: false }); + assert.deepEqual(worker.registry(Read).get("current")!(), { 1: false }); + assert.equal(server.epoch, worker.epoch); + assert.deepEqual(log, ["open", "open"]); +}); diff --git a/packages/chord/test/facet-loader.test.ts b/packages/chord/test/facet-loader.test.ts deleted file mode 100644 index d73372bad..000000000 --- a/packages/chord/test/facet-loader.test.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; -import { - type Context, - combineFacetLoaders, - createFacetHost, - createRemoteServiceBinding, - createStaticFacetLoader, - defineFacet, - defineService, - type FacetLoader, -} from "../src/index.ts"; -import { createLoopbackServiceTransport } from "./helpers.ts"; - -interface GenerationValue { - read(context: Context): Promise; -} - -const LocalGenerationValue = defineService("test.experimental.local-generation-value", { - local: true, -}); -const RemoteGenerationValue = defineService("test.experimental.remote-generation-value"); - -const firstFacet = defineFacet({ id: "first", setup() {} }); -const secondFacet = defineFacet({ id: "second", setup() {} }); - -describe("facet loader", () => { - test("combines loaded facets in loader order and disposes generations in reverse", async () => { - const trace: string[] = []; - const first: FacetLoader = { - async load() { - trace.push("load first"); - return { - facets: [firstFacet], - async dispose() { - trace.push("dispose first"); - }, - }; - }, - }; - const second: FacetLoader = { - async load() { - trace.push("load second"); - return { - facets: [secondFacet], - async dispose() { - trace.push("dispose second"); - }, - }; - }, - }; - - const loaded = await combineFacetLoaders([first, second]).load(); - expect(loaded.facets).toEqual([firstFacet, secondFacet]); - await loaded.dispose(); - await loaded.dispose(); - expect(trace).toEqual(["load first", "load second", "dispose second", "dispose first"]); - }); - - test("cleans up loaded facets when a later loader fails", async () => { - const dispose = vi.fn(async () => {}); - const failure = new Error("load failed"); - const first: FacetLoader = { - async load() { - return { facets: [firstFacet], dispose }; - }, - }; - const second: FacetLoader = { - async load() { - throw failure; - }, - }; - - await expect(combineFacetLoaders([first, second]).load()).rejects.toBe(failure); - expect(dispose).toHaveBeenCalledOnce(); - }); - - test("keeps local and RPC service handles stable when their provider facet reloads", async () => { - const trace: string[] = []; - let localValue: GenerationValue | undefined; - let generation = 0; - let reportReplacementStarted!: () => void; - const replacementStarted = new Promise((resolve) => { - reportReplacementStarted = resolve; - }); - let continueReplacement!: () => void; - const replacementCanContinue = new Promise((resolve) => { - continueReplacement = resolve; - }); - const consumer = defineFacet({ - id: "consumer", - setup(env) { - trace.push("setup consumer"); - localValue = env.use(LocalGenerationValue); - env.onActivate(async () => { - trace.push(`activate consumer:${await localValue!.read(BACKGROUND_CONTEXT)}`); - }); - env.onDeactivate(() => { - trace.push("deactivate consumer"); - }); - }, - }); - const loader: FacetLoader = { - async load() { - const name = generation++ === 0 ? "A" : "B"; - trace.push(`load ${name}`); - return { - facets: [ - defineFacet({ - id: "provider", - setup(env) { - trace.push(`setup provider ${name}`); - const implementation: GenerationValue = { - async read() { - return name; - }, - }; - env.provide(LocalGenerationValue, implementation); - env.provide(RemoteGenerationValue, implementation); - env.onActivate(async () => { - trace.push(`activate provider ${name}`); - if (name === "B") { - reportReplacementStarted(); - await replacementCanContinue; - } - }); - env.onDeactivate(() => { - trace.push(`deactivate provider ${name}`); - }); - }, - }), - ], - async dispose() { - trace.push(`unload ${name}`); - }, - }; - }, - }; - - const loadedA = await loader.load(); - const host = await createFacetHost({ facets: [consumer, ...loadedA.facets] }); - const originalLocalHandle = localValue!; - const localRead = originalLocalHandle.read; - const remoteServices = createRemoteServiceBinding({ - services: [RemoteGenerationValue], - transport: createLoopbackServiceTransport(host.services), - }); - const originalRemoteHandle = remoteServices.use(RemoteGenerationValue); - const remoteRead = originalRemoteHandle.read; - await remoteServices.ready(BACKGROUND_CONTEXT); - await expect(localRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - await expect(remoteRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - - await expect( - host.reload([ - defineFacet({ - id: "provider", - setup(env) { - env.provide(LocalGenerationValue, { - async read() { - return "invalid"; - }, - }); - }, - }), - ]), - ).rejects.toThrow("Reloaded facet provider must preserve its service requirements and provisions"); - await expect(localRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - await expect(remoteRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - - const loadedB = await loader.load(); - const reload = host.reload(loadedB.facets); - await replacementStarted; - await expect(localRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - await expect(remoteRead(BACKGROUND_CONTEXT)).resolves.toBe("A"); - continueReplacement(); - await reload; - await loadedA.dispose(); - - expect(localValue).toBe(originalLocalHandle); - expect(remoteServices.use(RemoteGenerationValue)).toBe(originalRemoteHandle); - await expect(localRead(BACKGROUND_CONTEXT)).resolves.toBe("B"); - await expect(remoteRead(BACKGROUND_CONTEXT)).resolves.toBe("B"); - - await remoteServices.dispose(BACKGROUND_CONTEXT); - await host.dispose(); - await loadedB.dispose(); - expect(trace).toEqual([ - "load A", - "setup consumer", - "setup provider A", - "activate provider A", - "activate consumer:A", - "load B", - "setup provider B", - "activate provider B", - "deactivate provider A", - "unload A", - "deactivate consumer", - "deactivate provider B", - "unload B", - ]); - }); - - test("rejects remote singleton member shape changes before reload cutover", async () => { - let retained: GenerationValue | undefined; - let providerDisposed = false; - const consumer = defineFacet({ - id: "shape-consumer", - setup(env) { - retained = env.use(RemoteGenerationValue); - }, - }); - const provider = defineFacet({ - id: "shape-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async read() { - return "A"; - }, - }); - env.onDeactivate(() => { - providerDisposed = true; - }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider] }); - - await expect( - host.reload([ - defineFacet({ - id: "shape-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async renamed() { - return "B"; - }, - } as unknown as GenerationValue); - }, - }), - ]), - ).rejects.toThrow("replacement must preserve its member shape"); - expect(providerDisposed).toBe(false); - await expect(retained!.read(BACKGROUND_CONTEXT)).resolves.toBe("A"); - - await host.dispose(); - expect(providerDisposed).toBe(true); - }); - - test("terminates the host when old cleanup fails after cutover", async () => { - const cleanupFailure = new Error("cleanup failed"); - const provider = (name: string, failCleanup: boolean) => - defineFacet({ - id: "cleanup-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async read() { - return name; - }, - }); - env.onDeactivate(() => { - if (failCleanup) throw cleanupFailure; - }); - }, - }); - const host = await createFacetHost({ facets: [provider("A", true)] }); - - await expect(host.reload([provider("B", false)])).rejects.toThrow("Facet reload failed after cutover"); - await expect(host.reload([])).rejects.toThrow("Facet host cannot reload while dead"); - await host.dispose(); - }); - - test("cleans failed candidate activation in reverse dependency order", async () => { - const failure = new Error("consumer activation failed"); - const trace: string[] = []; - const provider = (name: string) => - defineFacet({ - id: "ordered-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async read() { - return name; - }, - }); - env.onActivate(() => { - trace.push(`activate provider ${name}`); - }); - env.onDeactivate(() => { - trace.push(`deactivate provider ${name}`); - }); - }, - }); - const consumer = (name: string, fail: boolean) => - defineFacet({ - id: "ordered-consumer", - setup(env) { - env.use(RemoteGenerationValue); - env.onActivate(() => { - trace.push(`activate consumer ${name}`); - if (fail) throw failure; - }); - env.onDeactivate(() => { - trace.push(`deactivate consumer ${name}`); - }); - }, - }); - const host = await createFacetHost({ facets: [consumer("A", false), provider("A")] }); - trace.length = 0; - - await expect(host.reload([consumer("B", true), provider("B")])).rejects.toBe(failure); - expect(trace).toEqual([ - "activate provider B", - "activate consumer B", - "deactivate consumer B", - "deactivate provider B", - ]); - await host.dispose(); - }); - - test("terminates the host when replacement publication fails after cutover", async () => { - const publicationFailure = new Error("publication failed"); - const provider = (name: string) => - defineFacet({ - id: "publication-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async read() { - return name; - }, - }); - }, - }); - const host = await createFacetHost({ facets: [provider("A")] }); - const subscription = host.services.subscribe(RemoteGenerationValue.id, "singleton", () => { - throw publicationFailure; - }); - subscription.activate(); - - await expect(host.reload([provider("B")])).rejects.toThrow("Facet reload failed after cutover"); - await expect(host.reload([])).rejects.toThrow("Facet host cannot reload while dead"); - await host.dispose(); - }); - - test("keeps the old generation active when replacement activation fails before cutover", async () => { - const failure = new Error("replacement activation failed"); - const trace: string[] = []; - let retained: GenerationValue | undefined; - const consumer = defineFacet({ - id: "terminal-consumer", - setup(env) { - retained = env.use(RemoteGenerationValue); - env.onDeactivate(() => { - trace.push("deactivate consumer"); - }); - }, - }); - const provider = (name: string, fail: boolean) => - defineFacet({ - id: "terminal-provider", - setup(env) { - env.provide(RemoteGenerationValue, { - async read() { - return name; - }, - }); - env.onActivate(() => { - trace.push(`activate ${name}`); - if (fail) throw failure; - }); - env.onDeactivate(() => { - trace.push(`deactivate ${name}`); - }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider("A", false)] }); - await expect(retained!.read(BACKGROUND_CONTEXT)).resolves.toBe("A"); - - await expect(host.reload([provider("B", true)])).rejects.toBe(failure); - expect(trace).toEqual(["activate A", "activate B", "deactivate B"]); - await expect(retained!.read(BACKGROUND_CONTEXT)).resolves.toBe("A"); - await host.reload([]); - await host.dispose(); - expect(trace).toEqual(["activate A", "activate B", "deactivate B", "deactivate consumer", "deactivate A"]); - }); - - test("creates a reusable static loader", async () => { - const loader = createStaticFacetLoader([firstFacet]); - const first = await loader.load(); - const second = await loader.load(); - expect(first.facets).toEqual([firstFacet]); - expect(second.facets).toEqual([firstFacet]); - await Promise.all([first.dispose(), second.dispose()]); - }); -}); diff --git a/packages/chord/test/facets.test.ts b/packages/chord/test/facets.test.ts deleted file mode 100644 index 73d041065..000000000 --- a/packages/chord/test/facets.test.ts +++ /dev/null @@ -1,676 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; -import { - type Context, - createFacetHost, - createRemoteServiceBinding, - defineFacet, - defineService, - type MutableReplicatedState, - RemoteServiceProvider, - type RemoteServiceSource, - type ReplicatedState, -} from "../src/index.ts"; -import { createLoopbackServiceTransport } from "./helpers.ts"; - -interface Source { - read(context: Context): Promise; -} - -interface Projection { - read(context: Context): Promise; -} - -interface KeyedValue { - read(context: Context): Promise; -} - -interface Watched { - readonly state: ReplicatedState<{ value: number }>; -} - -interface HostValues { - readonly name: string; - readonly use: string; -} - -interface LocalKeyedValue { - readonly metadata: Map; - read(): string; -} - -interface LeftValue { - read(context: Context): Promise; -} - -interface RightValue { - read(context: Context): Promise; -} - -interface CombinedValue { - read(context: Context): Promise; -} - -const Source = defineService("test.experimental.source"); -const Projection = defineService("test.experimental.projection"); -const KeyedValue = defineService("test.experimental.keyed-value"); -const Watched = defineService("test.experimental.watched"); -const HostValues = defineService("test.experimental.host-values", { local: true }); -const LocalKeyedValue = defineService("test.experimental.local-keyed-value", { local: true }); -const LeftValue = defineService("test.experimental.left-value"); -const RightValue = defineService("test.experimental.right-value"); -const CombinedValue = defineService("test.experimental.combined-value"); - -describe("facet host", () => { - test("discovers setup dependencies before connecting stable service handles", async () => { - const trace: string[] = []; - let sourceHandle: Source | undefined; - const projection = defineFacet({ - id: "projection", - setup(env) { - trace.push("setup projection"); - sourceHandle = env.use(Source); - expect(() => sourceHandle!.read(BACKGROUND_CONTEXT)).toThrow( - "Facet projection service handles cannot be used while setting_up", - ); - env.provide(Projection, { - read: (context) => sourceHandle!.read(context), - }); - env.onActivate(() => { - trace.push("activate projection"); - }); - env.onDeactivate(() => { - trace.push("dispose projection"); - }); - }, - }); - const source = defineFacet({ - id: "source", - setup(env) { - trace.push("setup source"); - env.provide(Source, { - async read() { - return "value"; - }, - }); - env.onActivate(() => { - trace.push("activate source"); - }); - env.onDeactivate(() => { - trace.push("dispose source"); - }); - }, - }); - const host = await createFacetHost({ facets: [projection, source] }); - - expect(trace).toEqual(["setup projection", "setup source", "activate source", "activate projection"]); - expect(await sourceHandle!.read(BACKGROUND_CONTEXT)).toBe("value"); - expect(await host.services.use(Projection).read(BACKGROUND_CONTEXT)).toBe("value"); - - await host.dispose(); - expect(trace.slice(-2)).toEqual(["dispose projection", "dispose source"]); - }); - - test("connects keyed observations only when the observing facet activates", async () => { - const trace: string[] = []; - const observer = defineFacet({ - id: "observer", - setup(env) { - env.observe(KeyedValue, async (service, context) => { - trace.push(`observe ${await service.read(context)}`); - }); - env.onActivate(() => { - trace.push("activate observer"); - }); - }, - }); - const provider = defineFacet({ - id: "provider", - setup(env) { - const values = env.provideMany(KeyedValue); - env.onActivate(() => { - trace.push("activate provider"); - values.spawn("one", { - async read() { - return "one"; - }, - }); - }); - }, - }); - const host = await createFacetHost({ facets: [observer, provider] }); - await vi.waitFor(() => expect(trace).toContain("observe one")); - - expect(trace).toEqual(["activate provider", "activate observer", "observe one"]); - const remoteServices = createRemoteServiceBinding({ - services: [KeyedValue], - transport: createLoopbackServiceTransport(host.services), - }); - const remoteValues: string[] = []; - remoteServices.observe(KeyedValue, async (service, context) => { - remoteValues.push(await service.read(context)); - }); - await remoteServices.ready(BACKGROUND_CONTEXT); - await vi.waitFor(() => expect(remoteValues).toEqual(["one"])); - - await remoteServices.dispose(BACKGROUND_CONTEXT); - await host.dispose(); - }); - - test("routes remotely exposable keyed services through the host provider", async () => { - const observed: Array<{ service: KeyedValue; context: Context }> = []; - const consumer = defineFacet({ - id: "remote-keyed-consumer", - setup(env) { - env.observe(KeyedValue, (service, context) => { - observed.push({ service, context }); - }); - }, - }); - const provider = (value: string) => - defineFacet({ - id: "remote-keyed-provider", - setup(env) { - const values = env.provideMany(KeyedValue); - env.onActivate(() => { - values.spawn("current", { - async read() { - return value; - }, - }); - }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider("A")] }); - await vi.waitFor(() => expect(observed).toHaveLength(1)); - const first = observed[0]!; - await expect(first.service.read(first.context)).resolves.toBe("A"); - - await host.reload([provider("B")]); - await vi.waitFor(() => expect(observed).toHaveLength(2)); - expect(first.context.abortSignal?.aborted).toBe(true); - expect(() => first.service.read(first.context)).toThrow("observation is closed"); - await expect(observed[1]!.service.read(observed[1]!.context)).resolves.toBe("B"); - - await host.dispose(); - expect(observed[1]!.context.abortSignal?.aborted).toBe(true); - }); - - test("terminates the host when keyed replacement publication fails", async () => { - const failure = new Error("spawn publication failed"); - const provider = (value: string) => - defineFacet({ - id: "failing-keyed-provider", - setup(env) { - const values = env.provideMany(KeyedValue); - env.onActivate(() => { - values.spawn("current", { - async read() { - return value; - }, - }); - }); - }, - }); - const host = await createFacetHost({ facets: [provider("A")] }); - const subscription = host.services.subscribe(KeyedValue.id, "keyed", (update) => { - if (update.type === "spawned") throw failure; - }); - subscription.activate(); - - await expect(host.reload([provider("B")])).rejects.toThrow("Facet reload failed after cutover"); - await expect(host.reload([])).rejects.toThrow("Facet host cannot reload while dead"); - expect(() => host.services.use(KeyedValue)).toThrow("Remote service provider is disposed"); - await host.dispose(); - }); - - test("terminates the host when keyed retirement publication fails", async () => { - const failure = new Error("close publication failed"); - const provider = (value: string) => - defineFacet({ - id: "failing-keyed-retirement-provider", - setup(env) { - const values = env.provideMany(KeyedValue); - env.onActivate(() => { - values.spawn("current", { - async read() { - return value; - }, - }); - }); - }, - }); - const host = await createFacetHost({ facets: [provider("A")] }); - const subscription = host.services.subscribe(KeyedValue.id, "keyed", (update) => { - if (update.type === "closed") throw failure; - }); - subscription.activate(); - - await expect(host.reload([provider("B")])).rejects.toThrow("Facet reload failed after cutover"); - await expect(host.reload([])).rejects.toThrow("Facet host cannot reload while dead"); - await host.dispose(); - }); - - test("keeps unrestricted local keyed services process-local across provider reloads", async () => { - const observed: Array<{ service: LocalKeyedValue; context: Context }> = []; - const consumer = defineFacet({ - id: "local-keyed-consumer", - setup(env) { - env.observe(LocalKeyedValue, (service, context) => { - observed.push({ service, context }); - }); - }, - }); - const provider = (value: string) => - defineFacet({ - id: "local-keyed-provider", - setup(env) { - const values = env.provideMany(LocalKeyedValue); - env.onActivate(() => { - values.spawn("current", { - metadata: new Map([["value", value]]), - read: () => value, - }); - }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider("A")] }); - await vi.waitFor(() => expect(observed).toHaveLength(1)); - const first = observed[0]!; - expect(first.service.read()).toBe("A"); - expect(first.service.metadata.get("value")).toBe("A"); - expect(host.services.catalogue).not.toContainEqual({ serviceId: LocalKeyedValue.id, mode: "keyed" }); - expect(() => host.services.use(LocalKeyedValue)).toThrow("process-local"); - - await host.reload([provider("B")]); - await vi.waitFor(() => expect(observed).toHaveLength(2)); - expect(first.context.abortSignal?.aborted).toBe(true); - expect(() => first.service.read()).toThrow(`Keyed service ${LocalKeyedValue.id} observation is closed`); - expect(observed[1]!.service.read()).toBe("B"); - expect(observed[1]!.service.metadata.get("value")).toBe("B"); - - await host.dispose(); - expect(observed[1]!.context.abortSignal?.aborted).toBe(true); - }); - - test("keeps remotely exposable local state replicas stable across provider reloads", async () => { - const sources: MutableReplicatedState<{ value: number }>[] = []; - const revisions: number[] = []; - let watched: Watched | undefined; - const consumer = defineFacet({ - id: "state-consumer", - setup(env) { - watched = env.use(Watched); - env.onActivate(() => { - env.own(watched!.state.subscribe(({ value }) => revisions.push(value))); - }); - }, - }); - const provider = (value: number) => - defineFacet({ - id: "state-provider", - setup(env) { - const state = env.replicatedState({ value }); - sources.push(state); - env.provide(Watched, { state }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider(1)] }); - const retainedState = watched!.state; - expect(retainedState.value).toEqual({ value: 1 }); - expect(revisions).toEqual([1]); - - await host.reload([provider(2)]); - expect(watched!.state).toBe(retainedState); - expect(retainedState.value).toEqual({ value: 2 }); - expect(revisions).toEqual([1, 2]); - sources[0]!.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 3; - }); - expect(retainedState.value).toEqual({ value: 2 }); - expect(revisions).toEqual([1, 2]); - - await host.dispose(); - expect(() => retainedState.value).toThrow("Facet state-consumer service handles cannot be used while dead"); - }); - - test("scopes singleton service views to each facet lifecycle", async () => { - const consumerHandles: Source[] = []; - const cleanupValues: string[] = []; - let peerHandle: Source | undefined; - const consumer = (generation: string) => - defineFacet({ - id: "scoped-consumer", - setup(env) { - const source = env.use(Source); - expect(env.use(Source)).toBe(source); - consumerHandles.push(source); - expect(() => source.read(BACKGROUND_CONTEXT)).toThrow( - "Facet scoped-consumer service handles cannot be used while setting_up", - ); - env.onDeactivate(async () => { - cleanupValues.push(`${generation}:${await source.read(BACKGROUND_CONTEXT)}`); - }); - }, - }); - const peer = defineFacet({ - id: "peer-consumer", - setup(env) { - peerHandle = env.use(Source); - }, - }); - const provider = defineFacet({ - id: "scoped-provider", - setup(env) { - env.provide(Source, { - async read() { - return "value"; - }, - }); - }, - }); - const host = await createFacetHost({ facets: [consumer("A"), peer, provider] }); - expect(consumerHandles[0]).not.toBe(peerHandle); - const retainedOldRead = consumerHandles[0]!.read; - - await host.reload([consumer("B")]); - expect(cleanupValues).toEqual(["A:value"]); - expect(Object.is(consumerHandles[1], consumerHandles[0])).toBe(false); - expect(() => consumerHandles[0]!.read(BACKGROUND_CONTEXT)).toThrow( - "Facet scoped-consumer service handles cannot be used while dead", - ); - expect(() => retainedOldRead(BACKGROUND_CONTEXT)).toThrow( - "Facet scoped-consumer service handles cannot be used while dead", - ); - await expect(consumerHandles[1]!.read(BACKGROUND_CONTEXT)).resolves.toBe("value"); - await expect(peerHandle!.read(BACKGROUND_CONTEXT)).resolves.toBe("value"); - - await host.dispose(); - expect(cleanupValues).toEqual(["A:value", "B:value"]); - expect(() => consumerHandles[1]!.read(BACKGROUND_CONTEXT)).toThrow( - "Facet scoped-consumer service handles cannot be used while dead", - ); - expect(() => peerHandle!.read(BACKGROUND_CONTEXT)).toThrow( - "Facet peer-consumer service handles cannot be used while dead", - ); - }); - - test("owns resources registered during activation", async () => { - let state: MutableReplicatedState<{ value: number }> | undefined; - let deliveries = 0; - const consumer = defineFacet({ - id: "consumer", - setup(env) { - const watched = env.use(Watched); - env.onActivate(() => { - env.own( - watched.state.subscribe(() => { - deliveries += 1; - }), - ); - }); - }, - }); - const provider = defineFacet({ - id: "provider", - setup(env) { - state = env.replicatedState({ value: 0 }); - env.provide(Watched, { state }); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider] }); - expect(deliveries).toBe(1); - state!.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 1; - }); - expect(deliveries).toBe(2); - - await host.dispose(); - state!.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 2; - }); - expect(deliveries).toBe(2); - }); - - test("provides arbitrary host services through the facet graph", async () => { - const values: HostValues = { name: "session", use: "host value" }; - const consumer = defineFacet({ - id: "host-service-consumer", - setup(env) { - const hostValues = env.use(HostValues); - expect(() => hostValues.use).toThrow( - "Facet host-service-consumer service handles cannot be used while setting_up", - ); - env.onActivate(() => { - expect(hostValues).not.toBe(values); - expect(hostValues.name).toBe("session"); - expect(hostValues.use).toBe("host value"); - }); - }, - }); - const provider = defineFacet({ - id: "host-service-provider", - setup(env) { - env.provide(HostValues, values); - }, - }); - const host = await createFacetHost({ facets: [consumer, provider] }); - expect(() => host.services.use(HostValues)).toThrow("Service test.experimental.host-values is process-local"); - await host.dispose(); - }); - - test("combines connected services and facet-provided services in one host", async () => { - const leftProvider = new RemoteServiceProvider([LeftValue]); - leftProvider.provide(LeftValue, { - async read() { - return "left"; - }, - }); - const rightProvider = new RemoteServiceProvider([RightValue]); - rightProvider.provide(RightValue, { - async read() { - return "right"; - }, - }); - const leftNamespace = createRemoteServiceBinding({ - services: [LeftValue], - transport: createLoopbackServiceTransport(leftProvider), - bound: false, - }); - const rightNamespace = createRemoteServiceBinding({ - services: [RightValue], - transport: createLoopbackServiceTransport(rightProvider), - bound: false, - }); - const serviceSources = [ - { namespace: leftNamespace, provider: leftProvider }, - { namespace: rightNamespace, provider: rightProvider }, - ].map(({ namespace, provider }) => { - const ready = namespace.ready.bind(namespace); - return Object.assign(namespace, { - acceptsUnavailableServices: false, - async catalogue() { - return provider.catalogue; - }, - open() { - return namespace; - }, - async ready(context: Context) { - await namespace.rebind(true, context); - await ready(context); - }, - }); - }); - const facet = defineFacet({ - id: "combined", - setup(env) { - const left = env.use(LeftValue); - const right = env.use(RightValue); - env.provide(CombinedValue, { - async read(context) { - return `${await left.read(context)} ${await right.read(context)}`; - }, - }); - }, - }); - - const host = await createFacetHost({ facets: [facet], serviceSources }); - await expect(host.services.use(CombinedValue).read(BACKGROUND_CONTEXT)).resolves.toBe("left right"); - - await host.dispose(); - await Promise.all([leftNamespace.dispose(BACKGROUND_CONTEXT), rightNamespace.dispose(BACKGROUND_CONTEXT)]); - leftProvider.dispose(); - rightProvider.dispose(); - }); - - test("rejects a service offered by multiple sources", async () => { - const duplicate = { - acceptsUnavailableServices: false, - async catalogue() { - return [{ serviceId: LeftValue.id, mode: "singleton" as const }]; - }, - open() { - throw new Error("Ambiguous sources must not open"); - }, - } satisfies RemoteServiceSource; - const consumer = defineFacet({ - id: "duplicate-consumer", - setup(env) { - env.use(LeftValue); - }, - }); - await expect(createFacetHost({ facets: [consumer], serviceSources: [duplicate, duplicate] })).rejects.toThrow( - `Facet host service ${LeftValue.id} is offered by more than one source`, - ); - }); - - test("reopens source bindings from changed catalogues for a replacement generation", async () => { - const leftProvider = new RemoteServiceProvider([LeftValue]); - leftProvider.provide(LeftValue, { - async read() { - return "left"; - }, - }); - const rightProvider = new RemoteServiceProvider([RightValue]); - rightProvider.provide(RightValue, { - async read() { - return "right"; - }, - }); - let currentProvider = leftProvider; - let opened = 0; - let disposed = 0; - const source = { - acceptsUnavailableServices: false, - async catalogue() { - return currentProvider.catalogue; - }, - open(options) { - opened += 1; - const namespace = createRemoteServiceBinding({ - services: options.services, - transport: createLoopbackServiceTransport(currentProvider), - bound: false, - assertAccess: options.assertAccess, - onError: options.onError, - }); - const ready = namespace.ready.bind(namespace); - const dispose = namespace.dispose.bind(namespace); - return Object.assign(namespace, { - async ready(context: Context) { - await namespace.rebind(true, context); - await ready(context); - }, - async dispose(context: Context) { - disposed += 1; - await dispose(context); - }, - }); - }, - } satisfies RemoteServiceSource; - const values: string[] = []; - const leftConsumer = defineFacet({ - id: "left-consumer", - setup(env) { - const left = env.use(LeftValue); - env.onActivate(async () => { - values.push(await left.read(BACKGROUND_CONTEXT)); - }); - }, - }); - const first = await createFacetHost({ facets: [leftConsumer], serviceSources: [source] }); - await first.dispose(); - - currentProvider = rightProvider; - const rightConsumer = defineFacet({ - id: "right-consumer", - setup(env) { - const right = env.use(RightValue); - env.onActivate(async () => { - values.push(await right.read(BACKGROUND_CONTEXT)); - }); - }, - }); - const second = await createFacetHost({ facets: [rightConsumer], serviceSources: [source] }); - await second.dispose(); - - expect(values).toEqual(["left", "right"]); - expect(opened).toBe(2); - expect(disposed).toBe(2); - leftProvider.dispose(); - rightProvider.dispose(); - }); - - test("rejects missing dependencies, cycles, and asynchronous setup", async () => { - let activated = false; - const missing = defineFacet({ - id: "missing", - setup(env) { - env.use(Source); - env.onActivate(() => { - activated = true; - }); - }, - }); - await expect(createFacetHost({ facets: [missing] })).rejects.toThrow( - "Facet missing requires local/test.experimental.source/singleton, but no facet provides it", - ); - expect(activated).toBe(false); - - const first = defineFacet({ - id: "first", - setup(env) { - env.use(Projection); - env.provide(Source, { - async read() { - return "first"; - }, - }); - }, - }); - const second = defineFacet({ - id: "second", - setup(env) { - env.use(Source); - env.provide(Projection, { - async read() { - return "second"; - }, - }); - }, - }); - await expect(createFacetHost({ facets: [first, second] })).rejects.toThrow( - "Facet dependency cycle: first, second", - ); - - const asynchronous = defineFacet({ - id: "asynchronous", - async setup() { - await Promise.resolve(); - }, - }); - await expect(createFacetHost({ facets: [asynchronous] })).rejects.toThrow( - "Facet asynchronous setup must be synchronous", - ); - }); -}); diff --git a/packages/chord/test/fixtures/echo.ts b/packages/chord/test/fixtures/echo.ts new file mode 100644 index 000000000..e3e42b213 --- /dev/null +++ b/packages/chord/test/fixtures/echo.ts @@ -0,0 +1,10 @@ +import { Type } from "typebox"; +import { method, remote } from "../../src/index.ts"; + +export const Echo = remote("t.echo", { + version: 1, + input: Type.Object({}), + variants: { + main: { methods: { echo: method(Type.String(), Type.String()), pid: method(Type.Null(), Type.Number()) } }, + }, +}); diff --git a/packages/chord/test/fixtures/loader.ts b/packages/chord/test/fixtures/loader.ts new file mode 100644 index 000000000..034576231 --- /dev/null +++ b/packages/chord/test/fixtures/loader.ts @@ -0,0 +1,54 @@ +// Runs the node loader outside vitest: its module runner bypasses the Node resolve hooks the loader installs. +import assert from "node:assert/strict"; +import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { Type } from "typebox"; +import { createHost, local, method } from "../../src/index.ts"; +import { nodeLoader } from "../../src/loader.ts"; + +// Plugin modules live under the package so bare imports (typebox) resolve; removed afterwards. +const dir = fileURLToPath(new URL(`../.tmp-plugins-${process.pid}/`, import.meta.url)); +const chord = new URL("../../src/index.ts", import.meta.url).href; + +// The test's own copy of the plugin's contract: binding is by description, not object identity. +const Hello = local("t.hello", { + version: 1, + input: Type.Object({}), + members: { greet: method(Type.String(), Type.String()) }, +}); + +const facetSource = ` +import { Type } from "typebox"; +import { defineFacet, local, method } from ${JSON.stringify(chord)}; +import { greeting } from "./greeting.ts"; + +const Hello = local("t.hello", { + version: 1, + input: Type.Object({}), + members: { greet: method(Type.String(), Type.String()) }, +}); + +export default defineFacet((host) => { + host.serve(Hello, () => ({ greet: (name) => greeting + " " + name })); +}); +`; + +const writeGreeting = (text: string) => + writeFileSync(`${dir}greeting.ts`, `export const greeting = ${JSON.stringify(text)};\n`); + +try { + mkdirSync(dir, { recursive: true }); + writeFileSync(`${dir}facet.ts`, facetSource); + writeGreeting("v1"); + const host = createHost({ loader: nodeLoader, onError: () => {} }); + await host.load([`${dir}facet.ts`]); + const handle = host.get(Hello, {}); + assert.equal(await handle.greet("x"), "v1 x"); + writeGreeting("v2"); + const report = await host.reload([`${dir}facet.ts`]); + assert.deepEqual(report.changed, ["t.hello"]); + assert.equal(await handle.greet("x"), "v2 x"); + await host.close(); +} finally { + rmSync(dir, { recursive: true, force: true }); +} diff --git a/packages/chord/test/fixtures/worker.ts b/packages/chord/test/fixtures/worker.ts new file mode 100644 index 000000000..606379a1c --- /dev/null +++ b/packages/chord/test/fixtures/worker.ts @@ -0,0 +1,40 @@ +// Child host for manager tests. FIXTURE_MODE: normal | ignore-shutdown | fatal. +import { spawn } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { createHost, type IpcChannel, ipcLink } from "../../src/index.ts"; +import { childControl } from "../../src/manager.ts"; +import { Echo } from "./echo.ts"; + +const mode = process.env.FIXTURE_MODE ?? "normal"; +const control = childControl(); +if (mode === "fatal") { + control.fatal(new Error("fixture refuses to start")); + setTimeout(() => process.exit(1), 200); +} else { + // Before anything that owns data: the kernel lock (if the manager configured one). + await control.lock().catch(() => { + setTimeout(() => process.exit(1), 200); + return new Promise(() => {}); + }); + const host = createHost({ + name: process.env.HOST_NAME ?? "worker", + upstream: ipcLink(process as unknown as IpcChannel), + onError: () => {}, + }); + const init = (await control.init) as { greeting: string }; + host.serve(Echo, () => ({ + kind: "main", + echo: (text: string) => `${init.greeting} ${text} from ${process.pid}`, + pid: () => process.pid, + })); + if (process.env.GRANDCHILD_FILE) { + const grandchild = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" }); + writeFileSync(process.env.GRANDCHILD_FILE, String(grandchild.pid)); + } + control.onShutdown(async () => { + if (mode === "ignore-shutdown") return; + await host.close(); + process.exit(0); + }); + control.ready(); +} diff --git a/packages/chord/test/helpers.ts b/packages/chord/test/helpers.ts deleted file mode 100644 index b4259ef09..000000000 --- a/packages/chord/test/helpers.ts +++ /dev/null @@ -1 +0,0 @@ -export { createLoopbackServiceTransport } from "../src/services/loopback.ts"; diff --git a/packages/chord/test/host.test.ts b/packages/chord/test/host.test.ts new file mode 100644 index 000000000..93c094680 --- /dev/null +++ b/packages/chord/test/host.test.ts @@ -0,0 +1,561 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { + type Ctx, + cell, + createHost, + defineRegistry, + derivation, + forbidden, + type Handle, + type Host, + isChordError, + listener, + local, + method, + remote, + resource, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const quiet = (errors: unknown[] = []) => createHost({ onError: (error) => errors.push(error) }); + +const Counter = local("t.counter", { + version: 1, + input: Type.Object({ name: Type.String() }), + members: { + count: Type.Object({ n: Type.Number() }), + add: method(Type.Number(), Type.Number()), + }, +}); + +function serveCounter(host: Host, log: string[] = [], options: { linger?: number } = {}) { + host.serve( + Counter, + ({ name }, _principal, self) => { + log.push(`factory ${name}`); + const state = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + const res = self.resources({ + conn: resource({ + rev: 1, + open: async () => { + log.push(`open ${name}`); + return { name }; + }, + close: () => { + log.push(`close ${name}`); + }, + }), + }); + return { + count: state.count, + conn: res.conn, + add: (delta: number, ctx) => { + state.count.change(ctx, (draft) => { + draft.n += delta; + }); + return state.count.value.n; + }, + }; + }, + options, + ); +} + +describe("host: activation and lifetime", () => { + it("activates, goes live, binds members late, disposes after release", async () => { + const host = quiet(); + const log: string[] = []; + serveCounter(host, log); + const handle = host.get(Counter, { name: "a" }); + assert.equal(handle.status, "loading"); + assert.equal(handle.count.value, undefined); + const pending = handle.add(2); + await handle.ready; + assert.equal(handle.status, "live"); + assert.equal(await pending, 2); + assert.deepEqual(handle.count.value, { n: 2 }); + handle.release(); + assert.equal(handle.status, "released"); + await assert.rejects(handle.add(1), (error) => isChordError(error, "released")); + await tick(); + assert.deepEqual(log, ["factory a", "open a", "close a"]); + }); + + it("coalesces concurrent gets and keeps the record while held", async () => { + const host = quiet(); + const log: string[] = []; + serveCounter(host, log); + const a = host.get(Counter, { name: "x" }); + const b = host.get(Counter, { name: "x" }); + await Promise.all([a.ready, b.ready]); + assert.deepEqual(log, ["factory x", "open x"]); + a.release(); + await tick(); + assert.equal(await b.add(1), 1); + b.release(); + await tick(); + assert.deepEqual(log, ["factory x", "open x", "close x"]); + }); + + it("lingers after the last release and revives on a new get", async () => { + const host = quiet(); + const log: string[] = []; + serveCounter(host, log, { linger: 30 }); + const a = host.get(Counter, { name: "l" }); + await a.add(5); + a.release(); + const b = host.get(Counter, { name: "l" }); + await b.ready; + assert.deepEqual(b.count.value, { n: 5 }); + b.release(); + await new Promise((resolve) => setTimeout(resolve, 60)); + assert.deepEqual(log, ["factory l", "open l", "close l"]); + }); + + it("disposes an abandoned activation once it settles", async () => { + const host = quiet(); + const log: string[] = []; + let release = () => {}; + const Slow = local("t.slow", { version: 1, input: Type.Object({}) }); + host.serve(Slow, (_input, _principal, self) => { + self.resources({ + r: resource({ + rev: 1, + open: ({ signal }) => + new Promise((resolve) => { + release = () => resolve("r"); + signal.addEventListener("abort", () => log.push("abort seen")); + }), + close: () => { + log.push("closed"); + }, + }), + }); + return {}; + }); + const handle = host.get(Slow, {}); + await tick(); + handle.release(); + await tick(); + release(); + await tick(); + await tick(); + assert.deepEqual(log, ["abort seen", "closed"]); + const again = host.get(Slow, {}); + await tick(); + release(); + await again.ready; + assert.equal(again.status, "live"); + }); + + it("propagates activation failures to dependents and retries on the next get", async () => { + const host = quiet(); + let fail = true; + const Leaf = local("t.leaf", { version: 1, input: Type.Object({}) }); + const Parent = local("t.parent", { version: 1, input: Type.Object({}) }); + host.serve(Leaf, (_input, _principal, self) => { + self.resources({ + r: resource({ + rev: 1, + open: () => { + if (fail) throw new Error("boom"); + return 1; + }, + }), + }); + return {}; + }); + host.serve(Parent, (_input, _principal, self) => { + self.get(Leaf, {}); + return {}; + }); + const parent = host.get(Parent, {}); + // A plain error from a dependency surfaces as `unavailable`, with the original as cause. + await assert.rejects(parent.ready, (error) => isChordError(error, "unavailable") && /boom/.test(error.message)); + assert.equal(parent.status, "gone"); + fail = false; + const retry = host.get(Parent, {}); + await retry.ready; + assert.equal(retry.status, "live"); + }); + + it("unwinds declarations when a factory throws and rejects async factories", async () => { + const host = quiet(); + const Leaf = local("t.leaf2", { version: 1, input: Type.Object({}) }); + const Bad = local("t.bad", { version: 1, input: Type.Object({ async: Type.Boolean() }) }); + let leafLive = 0; + host.serve(Leaf, (_input, _principal, self) => { + self.resources({ + r: resource({ + rev: 1, + open: () => { + leafLive++; + }, + close: () => { + leafLive--; + }, + }), + }); + return {}; + }); + host.serve(Bad, ({ async }, _principal, self) => { + self.get(Leaf, {}); + if (async) return Promise.resolve({}) as never; + throw new Error("factory failed"); + }); + await assert.rejects(host.get(Bad, { async: false }).ready, /factory failed/); + await assert.rejects(host.get(Bad, { async: true }).ready, /synchronous/); + await tick(); + await tick(); + assert.equal(leafLive, 0); + }); + + it("rejects duplicate declaration names", async () => { + const host = quiet(); + const Dup = local("t.dup", { version: 1, input: Type.Object({}) }); + host.serve(Dup, (_input, _principal, self) => { + self.state({ a: cell(Type.Object({}), {}) }); + self.state({ a: cell(Type.Object({}), {}) }); + return {}; + }); + await assert.rejects(host.get(Dup, {}).ready, /duplicate state name a/); + }); + + it("detects hold cycles", async () => { + const host = quiet(); + const A = local("t.cycle-a", { version: 1, input: Type.Object({}) }); + const B = local("t.cycle-b", { version: 1, input: Type.Object({}) }); + host.serve(A, (_input, _principal, self) => { + self.get(B, {}); + return {}; + }); + host.serve(B, (_input, _principal, self) => { + self.get(A, {}); + return {}; + }); + await assert.rejects(host.get(A, {}).ready, /cycle/); + }); + + it("closes resources in reverse order, runs all steps and aggregates errors", async () => { + const errors: unknown[] = []; + const host = quiet(errors); + const log: string[] = []; + const Multi = local("t.multi", { version: 1, input: Type.Object({}) }); + host.serve(Multi, (_input, _principal, self) => { + const first = self.resources({ + a: resource({ rev: 1, open: () => "a", close: () => void log.push("close a") }), + }); + self.resources({ + b: resource({ + rev: 1, + deps: [first.a], + open: () => "b", + close: () => { + log.push("close b"); + throw new Error("b failed"); + }, + }), + c: resource({ + rev: 1, + open: () => "c", + close: () => { + log.push("close c"); + throw new Error("c failed"); + }, + }), + }); + return {}; + }); + const handle = host.get(Multi, {}); + await handle.ready; + handle.release(); + await tick(); + await tick(); + assert.equal(log.at(-1), "close a"); + assert.equal(log.length, 3); + assert.ok(errors[0] instanceof AggregateError); + assert.equal((errors[0] as AggregateError).errors.length, 2); + }); + + it("releases ctx.get holds when the call settles", async () => { + const host = quiet(); + const log: string[] = []; + serveCounter(host, log); + const value = await host.root(async (ctx) => { + const counter = ctx.get(Counter, { name: "c" }); + return counter.add(3, ctx); + }); + assert.equal(value, 3); + await tick(); + assert.deepEqual(log, ["factory c", "open c", "close c"]); + }); +}); + +describe("host: principals, variants and disposal", () => { + const Doc = remote("t.doc", { + version: 1, + input: Type.Object({ id: Type.String() }), + variants: { + viewer: { state: { view: Type.Object({ text: Type.String() }) } }, + editor: { + state: { view: Type.Object({ text: Type.String() }) }, + methods: { edit: method(Type.Object({ text: Type.String() }), Type.Object({ ok: Type.Boolean() })) }, + }, + }, + }); + + function serveDoc(host: Host, principals: unknown[] = []) { + host.serve(Doc, (_input, principal, self) => { + principals.push(principal); + const state = self.state({ view: cell(Type.Object({ text: Type.String() }), { text: "" }) }); + if ((principal as { role?: string }).role !== "editor") return { kind: "viewer", view: state.view }; + return { + kind: "editor", + view: state.view, + edit: ({ text }, ctx) => { + state.view.replace(ctx, { text }); + return text === "bad" ? ({ ok: "no" } as never) : { ok: true }; + }, + }; + }); + } + + it("keeps records per principal key and enforces variants", async () => { + const host = createHost({ principalKey: (principal) => (principal as { user: string }).user, onError: () => {} }); + const principals: unknown[] = []; + serveDoc(host, principals); + const viewer = host.get(Doc, { id: "1" }, { principal: { user: "v", role: "viewer" } }); + const editor = host.get(Doc, { id: "1" }, { principal: { user: "e", role: "editor" } }); + await Promise.all([viewer.ready, editor.ready]); + assert.equal(principals.length, 2); + assert.equal(viewer.kind, "viewer"); + assert.deepEqual(viewer.shape?.members, ["view"]); + await assert.rejects(viewer.edit({ text: "x" }), (error) => isChordError(error, "not_in_variant")); + assert.deepEqual(await editor.edit({ text: "x" }), { ok: true }); + assert.deepEqual(editor.view.value, { text: "x" }); + assert.deepEqual(viewer.view.value, { text: "" }); + }); + + it("validates remote args and results at the provider boundary", async () => { + const host = quiet(); + serveDoc(host); + const editor = host.get(Doc, { id: "2" }, { principal: { role: "editor" } }); + await assert.rejects( + editor.edit({ text: 1 } as never), + (error) => isChordError(error, "invalid") && error.reason === "args", + ); + await assert.rejects( + editor.edit({ text: "bad" }), + (error) => isChordError(error, "invalid") && error.reason === "result", + ); + }); + + it("forced disposal makes handles gone and aborts in-flight calls", async () => { + const host = quiet(); + let seenAbort = false; + const Slow = local("t.inflight", { + version: 1, + input: Type.Object({}), + members: { wait: method(Type.Null(), Type.Null()) }, + }); + host.serve(Slow, () => ({ + wait: (_args: null, ctx) => + new Promise((resolve) => + ctx.signal.addEventListener("abort", () => { + seenAbort = true; + resolve(null); + }), + ), + })); + const handle = host.get(Slow, {}, { principal: { user: "u" } }); + await handle.ready; + const call = handle.wait(null); + await tick(); + await host.invalidatePrincipal((principal) => (principal as { user: string }).user === "u"); + assert.equal(handle.status, "gone"); + await call; + assert.ok(seenAbort); + await assert.rejects(handle.wait(null), (error) => isChordError(error, "gone")); + const fresh = host.get(Slow, {}, { principal: { user: "u" } }); + await fresh.ready; + assert.equal(fresh.status, "live"); + }); + + it("state members are late-bound readables: values, then snapshot/reset/ops events", async () => { + const host = quiet(); + serveDoc(host); + const editor = host.get(Doc, { id: "3" }, { principal: { role: "editor" } }) as Handle; + const shapes: unknown[] = []; + editor.onShape((shape) => shapes.push(shape.kind)); + const events: unknown[] = []; + const values: unknown[] = []; + editor.view.events((event) => events.push(event.type === "ops" ? ["ops", event.ops] : [event.type, event.value])); + editor.view.subscribe((value) => values.push(value)); + await editor.ready; + await editor.edit({ text: "hi" }); + assert.deepEqual(shapes, ["editor"]); + assert.deepEqual(events, [ + ["snap", undefined], + ["reset", { text: "" }], + ["ops", [["r", { text: "hi" }]]], + ]); + assert.deepEqual(values, [undefined, { text: "" }, { text: "hi" }]); + }); +}); + +describe("host: registries", () => { + const Routes = defineRegistry("t.routes"); + + it("only eager records register; tables follow registration order", async () => { + const host = quiet(); + const RoutesA = local("t.routes-a", { version: 1, input: Type.Object({}) }); + const RoutesB = local("t.routes-b", { version: 1, input: Type.Object({}) }); + host.serve( + RoutesA, + (_input, _principal, self) => { + self.register(Routes, { "GET /a": "a", "GET /c": "c" }); + return {}; + }, + { eager: true }, + ); + host.serve( + RoutesB, + (_input, _principal, self) => { + self.register(Routes, { "GET /b": "b" }); + return {}; + }, + { eager: true }, + ); + const tables: string[][] = []; + host.registry(Routes).subscribe((table) => tables.push([...table.keys()])); + await tick(); + await tick(); + assert.deepEqual([...host.registry(Routes).table().keys()], ["GET /a", "GET /c", "GET /b"]); + assert.equal(host.registry(Routes).get("GET /b"), "b"); + assert.deepEqual(tables.at(-1), ["GET /a", "GET /c", "GET /b"]); + + const Lazy = local("t.lazy-register", { version: 1, input: Type.Object({}) }); + host.serve(Lazy, (_input, _principal, self) => { + self.register(Routes, { x: "x" }); + return {}; + }); + await assert.rejects(host.get(Lazy, {}).ready, /only eager/); + }); + + it("rejects conflicting registrations", async () => { + const errors: unknown[] = []; + const host = quiet(errors); + for (const name of ["t.conflict-a", "t.conflict-b"]) { + host.serve( + local(name, { version: 1, input: Type.Object({}) }), + (_input, _principal, self) => { + self.register(Routes, { same: name }); + return {}; + }, + { eager: true }, + ); + } + await tick(); + await tick(); + assert.ok(errors.some((error) => /registry_conflict/.test(String(error)))); + }); +}); + +describe("host: composition", () => { + const Source = local("t.source", { + version: 1, + input: Type.Object({}), + members: { n: Type.Number(), set: method(Type.Number(), Type.Null()) }, + }); + const Mirror = local("t.mirror", { + version: 1, + input: Type.Object({}), + members: { copy: Type.Number(), doubled: Type.Number() }, + }); + + it("handle state members are sources for listeners and derivations", async () => { + const host = quiet(); + host.serve(Source, (_input, _principal, self) => { + const state = self.state({ n: cell(Type.Number(), 1) }); + return { + n: state.n, + set: (value: number, ctx: Ctx) => { + state.n.replace(ctx, value); + return null; + }, + }; + }); + host.serve(Mirror, (_input, _principal, self) => { + const source = self.get(Source, {}); + const state = self.state({ copy: cell(Type.Number(), 0) }); + self.listen({ follow: listener(source.n, (value, ctx) => state.copy.replace(ctx, value ?? 0)) }); + const derived = self.derive({ doubled: derivation([source.n], (n) => (n ?? 0) * 2) }); + return { copy: state.copy, doubled: derived.doubled }; + }); + const mirror = host.get(Mirror, {}); + await mirror.ready; + await tick(); + assert.equal(mirror.copy.value, 1); + assert.equal(mirror.doubled.value, 2); + await host.get(Source, {}).set(5); + await tick(); + assert.equal(mirror.copy.value, 5); + assert.equal(mirror.doubled.value, 10); + }); + + it("ctx.get acts as the root principal at ingress and as the host inside objects", async () => { + const host = createHost({ principalKey: (principal) => JSON.stringify(principal), onError: () => {} }); + const seen: unknown[] = []; + const Who = local("t.who", { + version: 1, + input: Type.Object({}), + members: { ping: method(Type.Null(), Type.Null()) }, + }); + const Front = local("t.front", { + version: 1, + input: Type.Object({}), + members: { relay: method(Type.Null(), Type.Null()) }, + }); + host.serve(Who, (_input, principal) => { + seen.push(principal); + return { ping: () => null }; + }); + host.serve(Front, (_input, principal) => { + seen.push(principal); + return { relay: (_args: null, ctx: Ctx) => ctx.get(Who, {}).ping(null, ctx) }; + }); + await host.root((ctx) => ctx.get(Front, {}).relay(null, ctx), { principal: { user: "guest" } }); + assert.deepEqual(seen, [{ user: "guest" }, { host: true }]); + }); + + it("calling a handle inside a factory throws not_ready", async () => { + const host = quiet(); + host.serve(Source, (_input, _principal, self) => ({ + n: self.state({ n: cell(Type.Number(), 0) }).n, + set: () => null, + })); + const Eager = local("t.eager-call", { version: 1, input: Type.Object({}) }); + host.serve(Eager, (_input, _principal, self) => { + const source = self.get(Source, {}); + void source.set(1); + return {}; + }); + await assert.rejects(host.get(Eager, {}).ready, (error) => isChordError(error, "not_ready")); + }); + + it("a ChordError from a factory keeps its code through dependents", async () => { + const host = quiet(); + const Secret = local("t.secret", { version: 1, input: Type.Object({}) }); + const Page = local("t.page", { version: 1, input: Type.Object({}) }); + host.serve(Secret, () => { + throw forbidden("not for you"); + }); + host.serve(Page, (_input, _principal, self) => { + self.get(Secret, {}); + return {}; + }); + await assert.rejects(host.get(Page, {}).ready, (error) => isChordError(error, "forbidden")); + }); +}); diff --git a/packages/chord/test/link.test.ts b/packages/chord/test/link.test.ts new file mode 100644 index 000000000..6b4c09170 --- /dev/null +++ b/packages/chord/test/link.test.ts @@ -0,0 +1,343 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { + type Ctx, + cell, + createHost, + type Frame, + type Host, + isChordError, + memoryLinkPair, + method, + type Principal, + remote, + resource, + type StateEvent, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const Counter = remote("t.counter", { + version: 1, + input: Type.Object({ name: Type.String() }), + variants: { + main: { + state: { count: Type.Object({ n: Type.Number() }) }, + methods: { + add: method(Type.Number(), Type.Number()), + who: method(Type.Null(), Type.Unknown()), + }, + }, + }, +}); + +/** Serves Counter and records the principal of every record it creates. */ +function serveCounter(host: Host, principals: unknown[] = []) { + host.serve(Counter, (_input, principal, self) => { + principals.push(principal); + const state = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + return { + kind: "main", + count: state.count, + add: (delta, ctx) => { + state.count.change(ctx, (draft) => { + draft.n += delta; + }); + return state.count.value.n; + }, + who: () => principal, + }; + }); +} + +/** A server and a worker connected by a memory link; the server accepts the worker as `{ worker: name }`. */ +function serverWithWorker(workerName = "session:S1") { + const errors: unknown[] = []; + const server = createHost({ name: "server", onError: (error) => errors.push(error) }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: workerName }, kind: "host", name: workerName }); + const worker = createHost({ name: workerName, upstream: up, onError: (error) => errors.push(error) }); + return { server, worker, down, up, errors }; +} + +describe("links: calls and state across hosts", () => { + it("a child calls the server; the server stamps the child's principal", async () => { + const { server, worker } = serverWithWorker(); + const principals: unknown[] = []; + serveCounter(server, principals); + const result = await worker.root(async (ctx) => { + const counter = ctx.get(Counter, { name: "c" }, { at: "server" }); + await counter.ready; + assert.equal(counter.kind, "main"); + return [await counter.add(2, ctx), await counter.who(null, ctx)]; + }); + assert.deepEqual(result, [2, { worker: "session:S1" }]); + assert.deepEqual(principals, [{ worker: "session:S1" }]); + }); + + it("state members: snapshot, then delta-encoded ops; values decode to the provider's", async () => { + const { server, worker } = serverWithWorker(); + serveCounter(server); + const counter = worker.get(Counter, { name: "s" }, { at: "server" }); + const events: StateEvent[] = []; + const values: unknown[] = []; + counter.count.events((event) => events.push(event)); + counter.count.subscribe((value) => values.push(value)); + await counter.ready; + await counter.add(1); + await counter.add(1); + await tick(); + assert.deepEqual(counter.count.value, { n: 2 }); + assert.deepEqual( + events.map((event) => event.type), + ["snap", "reset", "ops", "ops"], + ); + assert.deepEqual(values, [undefined, { n: 0 }, { n: 1 }, { n: 2 }]); + }); + + it("the server reaches a connected child by name", async () => { + const { server, worker } = serverWithWorker("w1"); + const principals: unknown[] = []; + serveCounter(worker, principals); + await tick(); + const counter = server.get(Counter, { name: "x" }, { at: "w1" }); + assert.equal(await counter.add(5), 5); + // The parent's own requests act as `{ host: }` on the child. + assert.deepEqual(principals, [{ host: "server" }]); + }); + + it("rejects local types and unknown hosts", async () => { + const { worker } = serverWithWorker(); + const missing = worker.get(Counter, { name: "m" }, { at: "server" }); + await assert.rejects(missing.ready, (error) => isChordError(error, "unavailable")); + }); +}); + +interface Cluster { + readonly server: Host; + readonly workers: Map; + readonly log: string[]; + browser( + principal: Principal, + options?: { restamp?: (principal: Principal) => Principal | undefined }, + ): { + host: Host; + up: ReturnType[1]; + reconnect(): void; + }; + allow: (principal: Principal, host: string) => boolean; +} + +/** Server routing `session:*` to workers it spawns on demand (memory links); browsers connect as clients. */ +function cluster(options: { pingInterval?: number; pingTimeout?: number } = {}): Cluster { + const log: string[] = []; + const server = createHost({ name: "server", onError: () => {} }); + const workers = new Map(); + const state: Cluster = { + server, + workers, + log, + allow: () => true, + browser(principal, browserOptions = {}) { + const [down, up] = memoryLinkPair(); + server.accept(down, { principal, restamp: browserOptions.restamp, ...options }); + const host = createHost({ + name: "browser", + kind: "client", + upstream: up, + upstreamOptions: options, + onError: () => {}, + }); + const handle = { host, up, reconnect: () => {} }; + handle.reconnect = () => { + const [nextDown, nextUp] = memoryLinkPair(); + server.accept(nextDown, { principal }); + handle.up = nextUp; + host.connect(nextUp); + }; + return handle; + }, + }; + server.route("session:", { + authorize: (principal, host) => state.allow(principal, host), + resolve: async (name) => { + const [down, up] = memoryLinkPair(); + const worker = createHost({ name, upstream: up, upstreamOptions: options, onError: () => {} }); + worker.serve(Counter, (input, principal, self) => { + self.resources({ + probe: resource({ + rev: 1, + open: () => void log.push(`open ${input.name}`), + close: () => void log.push(`close ${input.name}`), + }), + }); + const counter = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + return { + kind: "main", + count: counter.count, + add: (delta: number, ctx: Ctx) => { + counter.count.change(ctx, (draft) => { + draft.n += delta; + }); + return counter.count.value.n; + }, + who: () => principal, + }; + }); + workers.set(name, worker); + return down; + }, + }); + return state; +} + +describe("links: relay, stamping, lifetime", () => { + it("relays browser → server → worker; the worker sees the browser's stamped principal", async () => { + const c = cluster(); + const guest = { guest: "g1" }; + const browser = c.browser(guest); + const counter = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + assert.equal(await counter.add(3), 3); + assert.deepEqual(await counter.who(null), guest); + const values: unknown[] = []; + counter.count.subscribe((value) => values.push(value)); + await counter.add(1); + await tick(); + assert.deepEqual(values.at(-1), { n: 4 }); + }); + + it("ignores payload principals from clients and from child hosts", async () => { + const c = cluster(); + const guest = { guest: "g1" }; + const [down, raw] = memoryLinkPair(); + c.server.accept(down, { principal: guest }); + const frames: Frame[] = []; + raw.receive((frame) => frames.push(frame)); + raw.send({ t: "hello", name: "raw", kind: "client", epoch: 0 }); + raw.send({ + t: "hold", + id: 1, + at: "session:S1", + token: Counter.description, + input: { name: "f" }, + principal: { owner: true }, + }); + raw.send({ t: "call", id: 2, hold: 1, member: "who", args: null }); + for (let i = 0; i < 50 && !frames.some((frame) => frame.t === "result"); i++) await sleep(5); + const result = frames.find((frame) => frame.t === "result"); + assert.deepEqual(result?.t === "result" ? result.value : undefined, guest); + + // A child host cannot forward principals upward either: the server stamps it. + const { server, down: _down, up } = serverWithWorker("w9"); + serveCounter(server); + const childFrames: Frame[] = []; + up.receive((frame) => childFrames.push(frame)); + up.send({ + t: "hold", + id: 7, + at: "server", + token: Counter.description, + input: { name: "f" }, + principal: { owner: true }, + }); + up.send({ t: "call", id: 8, hold: 7, member: "who", args: null }); + for (let i = 0; i < 50 && !childFrames.some((frame) => frame.t === "result"); i++) await sleep(5); + const childResult = childFrames.find((frame) => frame.t === "result"); + assert.deepEqual(childResult?.t === "result" ? childResult.value : undefined, { worker: "w9" }); + }); + + it("authorizes before resolving and again after", async () => { + const c = cluster(); + c.allow = (principal) => (principal as { guest: string }).guest !== "blocked"; + const blocked = c.browser({ guest: "blocked" }); + await assert.rejects(blocked.host.get(Counter, { name: "a" }, { at: "session:S1" }).ready, (error) => + isChordError(error, "forbidden"), + ); + assert.equal(c.workers.size, 0); + let calls = 0; + c.allow = () => ++calls === 1; + const flaky = c.browser({ guest: "flaky" }); + await assert.rejects(flaky.host.get(Counter, { name: "a" }, { at: "session:S2" }).ready, (error) => + isChordError(error, "forbidden"), + ); + assert.equal(calls, 2); + }); + + it("a browser disconnect releases exactly its holds; the worker link stays up", async () => { + const c = cluster(); + const one = c.browser({ guest: "g1" }); + const two = c.browser({ guest: "g2" }); + await one.host.get(Counter, { name: "a" }, { at: "session:S1" }).add(1); + const other = two.host.get(Counter, { name: "b" }, { at: "session:S1" }); + await other.add(1); + one.up.close(); + for (let i = 0; i < 20 && !c.log.includes("close a"); i++) await tick(); + assert.ok(c.log.includes("close a")); + assert.ok(!c.log.includes("close b")); + assert.equal(await other.add(1), 2); + }); + + it("a dead link (no pongs) is closed and everything held through it is released", async () => { + const c = cluster({ pingInterval: 10, pingTimeout: 40 }); + const browser = c.browser({ guest: "g1" }); + await browser.host.get(Counter, { name: "a" }, { at: "session:S1" }).add(1); + browser.up.sever(); + for (let i = 0; i < 40 && !c.log.includes("close a"); i++) await sleep(10); + assert.ok(c.log.includes("close a")); + }); + + it("checks contract tokens across links: contract_mismatch for hosts, upgrade_required for browsers", async () => { + const CounterV2 = remote("t.counter", { + version: 2, + input: Type.Object({ name: Type.String() }), + variants: { main: { methods: { inc: method(Type.Null(), Type.Number()) } } }, + }); + const { server, worker } = serverWithWorker(); + server.serve(CounterV2, () => ({ kind: "main", inc: () => 1 })); + await assert.rejects(worker.get(Counter, { name: "x" }, { at: "server" }).ready, (error) => + isChordError(error, "contract_mismatch"), + ); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + await assert.rejects(browser.get(Counter, { name: "x" }, { at: "server" }).ready, (error) => + isChordError(error, "upgrade_required"), + ); + }); + + it("invalidatePrincipal re-stamps the link and propagates to workers", async () => { + const c = cluster(); + const browser = c.browser({ guest: "g1", revision: 1 }, { restamp: (p) => ({ ...(p as object), revision: 2 }) }); + const counter = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + await counter.add(1); + await c.server.invalidatePrincipal({ guest: "g1" }); + // The relay under the old principal is closed at once; the worker disposes its record after. + for (let i = 0; i < 20 && counter.status !== "gone"; i++) await tick(); + assert.equal(counter.status, "gone"); + for (let i = 0; i < 20 && !c.log.includes("close a"); i++) await tick(); + assert.ok(c.log.includes("close a")); + const again = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + assert.deepEqual(await again.who(null), { guest: "g1", revision: 2 }); + }); + + it("after a reconnect, state subscribers get a reset and calls work again", async () => { + const c = cluster(); + const browser = c.browser({ guest: "g1" }); + const counter = browser.host.get(Counter, { name: "r" }, { at: "session:S1" }); + const events: string[] = []; + counter.count.events((event) => events.push(event.type)); + await counter.add(2); + browser.up.close(); + await tick(); + await tick(); + assert.equal(counter.status, "loading"); + browser.reconnect(); + assert.equal(await counter.add(1), 1); + await tick(); + // The old hold was released with the link, so the worker made a fresh record; the reset carries its value. + assert.deepEqual(counter.count.value, { n: 1 }); + assert.ok(events.filter((type) => type === "reset").length >= 2); + }); +}); diff --git a/packages/chord/test/links-review.test.ts b/packages/chord/test/links-review.test.ts new file mode 100644 index 000000000..9f78bc294 --- /dev/null +++ b/packages/chord/test/links-review.test.ts @@ -0,0 +1,527 @@ +// Regression tests for Astra's phase 1b review (findings #1-#11) and the shared-worker test gap. +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { + type Ctx, + cell, + createHost, + type Facet, + type FacetLoader, + type Frame, + type Host, + isChordError, + type Link, + memoryLinkPair, + method, + type Principal, + remote, + resource, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +async function until(check: () => boolean, attempts = 100): Promise { + for (let i = 0; i < attempts && !check(); i++) await sleep(2); + assert.ok(check(), "condition not reached"); +} + +function latch() { + let open = () => {}; + const promise = new Promise((resolve) => { + open = resolve; + }); + return { promise, open }; +} + +/** A link that logs what it sends and can drop a frame by closing instead. */ +class WatchedLink implements Link { + readonly inner: Link; + readonly sent: Frame[] = []; + closeOn: ((frame: Frame) => boolean) | undefined; + constructor(inner: Link) { + this.inner = inner; + } + get closed(): boolean { + return this.inner.closed; + } + send(frame: Frame): void { + this.sent.push(frame); + if (this.closeOn?.(frame)) { + this.inner.close("dropped"); + return; + } + this.inner.send(frame); + } + receive(handler: (frame: Frame) => void): void { + this.inner.receive(handler); + } + onClose(handler: (reason: string) => void): void { + this.inner.onClose(handler); + } + close(reason?: string): void { + this.inner.close(reason); + } +} + +const Counter = remote("t.counter", { + version: 1, + input: Type.Object({ name: Type.String() }), + variants: { + main: { + state: { count: Type.Object({ n: Type.Number() }) }, + methods: { + add: method(Type.Number(), Type.Number()), + who: method(Type.Null(), Type.Unknown()), + slow: method(Type.Null(), Type.Null()), + }, + }, + }, +}); + +/** A server routing `session:*` to workers it spawns once per name; the server→worker link is watched. */ +function cluster() { + const log: string[] = []; + const principals: unknown[] = []; + const server = createHost({ name: "server", onError: () => {} }); + const workers = new Map(); + const state = { + server, + log, + principals, + workers, + resolveGate: undefined as Promise | undefined, + slowGate: latch(), + browser(principal: Principal, restamp?: (p: Principal) => Principal | undefined) { + const [down, up] = memoryLinkPair(); + server.accept(down, { principal, restamp }); + return { host: createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }), up }; + }, + raw(principal: Principal) { + const [down, raw] = memoryLinkPair(); + server.accept(down, { principal }); + const frames: Frame[] = []; + raw.receive((frame) => frames.push(frame)); + raw.send({ t: "hello", name: "raw", kind: "client", epoch: 0 }); + return { raw, frames }; + }, + }; + server.route("session:", { + authorize: () => true, + resolve: async (name) => { + await state.resolveGate; + const existing = workers.get(name); + if (existing !== undefined) return existing.link; + const [down, up] = memoryLinkPair(); + const link = new WatchedLink(down); + const worker = createHost({ name, upstream: up, onError: () => {} }); + worker.serve(Counter, (input, principal, self) => { + principals.push(principal); + self.resources({ + probe: resource({ + rev: 1, + open: () => void log.push(`open ${input.name}`), + close: () => void log.push(`close ${input.name}`), + }), + }); + const counter = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 0 }) }); + return { + kind: "main", + count: counter.count, + add: (delta: number, ctx: Ctx) => { + counter.count.change(ctx, (draft) => { + draft.n += delta; + }); + return counter.count.value.n; + }, + who: () => principal, + slow: async (_args: null, ctx: Ctx) => { + ctx.signal.addEventListener("abort", () => log.push("slow aborted")); + await state.slowGate.promise; + return null; + }, + }; + }); + workers.set(name, { host: worker, link }); + return link; + }, + }); + return state; +} + +const internals = (host: Host) => + host as unknown as { + connections: Set<{ + role: string; + peer: { kind: string; name?: string }; + relayedHolds: Map; + relayedSubs: Map; + relayedCalls: Map; + }>; + reloading: boolean; + }; + +describe("links review: identity and stamping", () => { + it("#1 the acceptor decides what a peer is: a client announcing itself as a host is cut off", async () => { + const c = cluster(); + const [down, raw] = memoryLinkPair(); + c.server.accept(down, { principal: { guest: "g" } }); + raw.receive(() => {}); + raw.send({ t: "hello", name: "session:S9", kind: "host", epoch: 1 }); + await until(() => raw.closed); + // A host link must have a name and its hello must match it. + assert.throws(() => c.server.accept(memoryLinkPair()[0], { principal: {}, kind: "host" }), /needs its name/); + }); + + it("#1 root_end is honored only from the link the root came over", async () => { + const server = createHost({ name: "server", onError: () => {} }); + const Echo = remote("t.echo", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { root: method(Type.Null(), Type.String()) } } }, + }); + const facet = (version: number): Facet => ({ + name: "echo", + setup: (api) => + api.serve( + Echo, + (_input, _principal, self) => { + self.state({ n: cell(Type.Number(), 0) }); + return { kind: "main", root: (_args: null, ctx: Ctx) => ctx.rootId }; + }, + { state: { version } }, + ), + }); + server.install(facet(1)); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "w" }, kind: "host", name: "w" }); + const worker = createHost({ name: "w", upstream: up, onError: () => {} }); + const go = latch(); + let rootId = ""; + const pinned = worker.root(async (ctx) => { + rootId = (await ctx.get(Echo, {}, { at: "server" }).root(null, ctx)) as string; + await go.promise; + }); + await until(() => rootId !== ""); + // A client forging the worker root's end must not release its lease. + const [clientDown, raw] = memoryLinkPair(); + server.accept(clientDown, { principal: { guest: "g" } }); + raw.receive(() => {}); + raw.send({ t: "hello", name: "raw", kind: "client", epoch: 0 }); + raw.send({ t: "root_end", rootId }); + await sleep(10); + await assert.rejects(server.reload([facet(2)], { gateDeadline: 40 }), (error) => { + assert.ok(isChordError(error, "aborted") && error.reason === "gate_deadline"); + assert.ok(error.message.includes(rootId)); + return true; + }); + go.open(); + await pinned; + }); + + it("#2 a revocation while the route is still resolving closes the relay; no worker record is made", async () => { + const c = cluster(); + const gate = latch(); + c.resolveGate = gate.promise; + const browser = c.browser({ guest: "g", revision: 1 }, (p) => ({ ...(p as object), revision: 2 })); + const counter = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + await sleep(5); + await c.server.invalidatePrincipal({ guest: "g" }); + await until(() => counter.status === "gone"); + gate.open(); + await sleep(10); + assert.deepEqual(c.principals, []); + const again = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + assert.deepEqual(await again.who(null), { guest: "g", revision: 2 }); + }); + + it("#10 malformed frames and duplicate ids close the link", async () => { + const c = cluster(); + const first = c.raw({ guest: "g" }); + first.raw.send({ t: "call", hold: 1 } as never); + await until(() => first.raw.closed); + const second = c.raw({ guest: "g" }); + const token = Counter.description; + second.raw.send({ t: "hold", id: 1, at: "session:S1", token, input: { name: "a" } }); + second.raw.send({ t: "hold", id: 1, at: "session:S1", token, input: { name: "b" } }); + await until(() => second.raw.closed); + }); +}); + +describe("links review: relay ownership", () => { + it("#7 a browser disconnect cancels its call downstream", async () => { + const c = cluster(); + const browser = c.browser({ guest: "g" }); + const counter = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + await counter.ready; + void counter.slow(null).catch(() => {}); + await sleep(10); + browser.up.close(); + await until(() => c.log.includes("slow aborted")); + const down = [...internals(c.server).connections].find((connection) => connection.peer.name === "session:S1")!; + assert.equal(down.relayedCalls.size, 0); + }); + + it("#9 a subscription dropped while the route resolves is never forwarded", async () => { + const c = cluster(); + const gate = latch(); + c.resolveGate = gate.promise; + const { raw } = c.raw({ guest: "g" }); + raw.send({ t: "hold", id: 1, at: "session:S1", token: Counter.description, input: { name: "a" } }); + raw.send({ t: "sub", id: 2, hold: 1, member: "count" }); + raw.send({ t: "unsub", id: 2 }); + await sleep(5); + gate.open(); + await until(() => c.workers.get("session:S1")?.link.sent.some((frame) => frame.t === "hold") === true); + await sleep(10); + assert.ok(!c.workers.get("session:S1")!.link.sent.some((frame) => frame.t === "sub")); + }); + + it("#9 a release while the route resolves settles waiting calls and forwards nothing", async () => { + const c = cluster(); + const gate = latch(); + c.resolveGate = gate.promise; + const { raw, frames } = c.raw({ guest: "g" }); + raw.send({ t: "hold", id: 1, at: "session:S1", token: Counter.description, input: { name: "a" } }); + raw.send({ t: "call", id: 2, hold: 1, member: "add", args: 1 }); + await sleep(5); + raw.send({ t: "release", id: 1 }); + await until(() => frames.some((frame) => frame.t === "result" && frame.id === 2)); + const result = frames.find((frame) => frame.t === "result" && frame.id === 2); + assert.ok(result?.t === "result" && result.error !== undefined); + gate.open(); + await sleep(10); + assert.ok(!(c.workers.get("session:S1")?.link.sent ?? []).some((frame) => frame.t === "hold")); + }); + + it("#9 an origin disconnect removes its relayed holds and subscriptions downstream", async () => { + const c = cluster(); + const browser = c.browser({ guest: "g" }); + const counter = browser.host.get(Counter, { name: "a" }, { at: "session:S1" }); + counter.count.subscribe(() => {}); + await counter.add(1); + const down = [...internals(c.server).connections].find((connection) => connection.peer.name === "session:S1")!; + assert.equal(down.relayedSubs.size, 1); + browser.up.close(); + await until(() => down.relayedHolds.size === 0 && down.relayedSubs.size === 0); + await until(() => c.log.includes("close a")); + }); + + it("two browsers share one worker connection; one leaving releases only its own holds", async () => { + const c = cluster(); + const one = c.browser({ guest: "g1" }); + const two = c.browser({ guest: "g2" }); + const a = one.host.get(Counter, { name: "same" }, { at: "session:S1" }); + const b = two.host.get(Counter, { name: "same" }, { at: "session:S1" }); + assert.equal(await a.add(1), 1); + assert.equal(await b.add(5), 5); + assert.equal(c.workers.size, 1); + assert.deepEqual(c.principals, [{ guest: "g1" }, { guest: "g2" }]); + one.up.close(); + await until(() => c.log.filter((entry) => entry === "close same").length === 1); + assert.equal(await b.add(1), 6); + }); + + it("#11 snapshots through an accepts adapter are validated against the consumer's schema", async () => { + const DocV1 = remote("t.doc", { + version: 1, + input: Type.Object({}), + variants: { main: { state: { view: Type.Object({ text: Type.String() }) } } }, + }); + const DocV2 = remote("t.doc", { + version: 2, + input: Type.Object({}), + variants: { main: { state: { view: Type.Object({ body: Type.String() }) } } }, + }); + const server = createHost({ name: "server", onError: () => {} }); + server.serve( + DocV2, + (_input, _principal, self) => ({ + kind: "main", + view: self.state({ view: cell(Type.Object({ body: Type.String() }), { body: "hi" }) }).view, + }), + { + accepts: { + 1: (current) => { + const view = current.view as { + value: { body: string }; + events: (fn: (event: never) => void) => () => void; + }; + const map = (value: { body: string }) => ({ text: value.body }); + return { + kind: "main", + view: { + get value() { + return map(view.value); + }, + subscribe: (fn: (value: unknown) => void) => view.events(() => fn(map(view.value))), + events: (fn: (event: unknown) => void) => + view.events((event: { type: string }) => + fn({ type: event.type === "ops" ? "reset" : event.type, value: map(view.value) }), + ), + }, + }; + }, + }, + }, + ); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + const doc = browser.get(DocV1, {}, { at: "server" }); + doc.view.subscribe(() => {}); + await until(() => doc.view.value !== undefined); + assert.deepEqual(doc.view.value, { text: "hi" }); + }); +}); + +describe("links review: cross-host reload", () => { + const Greeter = remote("t.greeter", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + }); + const greeterFacet = (greeting: string): Facet => ({ + name: "greeter", + setup: (api) => api.serve(Greeter, () => ({ kind: "main", greet: (name: string) => `${greeting} ${name}` })), + }); + + function setup(options: { slowLoad?: Promise } = {}) { + const modules = new Map([["greeter", greeterFacet("v1")]]); + const loader: FacetLoader = async (names) => { + await options.slowLoad; + return names.map((name) => modules.get(String(name))!); + }; + const server = createHost({ name: "server", onError: () => {} }); + const connect = () => { + const [down, up] = memoryLinkPair(); + const watched = new WatchedLink(down); + server.accept(watched, { principal: { worker: "w" }, kind: "host", name: "session:S1" }); + return { watched, up }; + }; + const first = connect(); + const worker = createHost({ name: "session:S1", upstream: first.up, loader, onError: () => {} }); + worker.install(modules.get("greeter")!); + return { server, worker, modules, link: first.watched, connect }; + } + + it("#3 an abort while the child is still loading modules frees its reload slot; a later reload works", async () => { + const gate = latch(); + const { server, worker, modules } = setup({ slowLoad: gate.promise }); + await tick(); + modules.set("greeter", greeterFacet("v2")); + await assert.rejects(server.reload([], { remote: { "session:": ["greeter"] }, voteDeadline: 30 }), (error) => + isChordError(error, "aborted"), + ); + gate.open(); + await until(() => !internals(worker).reloading); + await server.reload([], { remote: { "session:": ["greeter"] } }); + assert.equal(await server.get(Greeter, {}, { at: "session:S1" }).greet("x"), "v2 x"); + }); + + it("#3 the parent link dying while the child prepares unwinds the child", async () => { + const gate = latch(); + const { server, worker, modules, link } = setup({ slowLoad: gate.promise }); + await tick(); + modules.set("greeter", greeterFacet("v2")); + const reload = server.reload([], { remote: { "session:": ["greeter"] } }); + await sleep(5); + link.close("gone"); + await assert.rejects(reload); + gate.open(); + await until(() => !internals(worker).reloading); + assert.equal(worker.epoch, 1); + }); + + it("#4 tasks the commit starts cannot reach a child before the child's commit", async () => { + const { server, modules } = setup(); + await tick(); + const results: unknown[] = []; + const Caller = remote("t.caller", { version: 1, input: Type.Object({}), variants: { main: {} } }); + const callerFacet = (active: boolean): Facet => ({ + name: "caller", + setup: (api) => + api.serve( + Caller, + (_input, _principal, self) => { + const greeter = self.get(Greeter, {}, { at: "session:S1" }); + if (active) { + self.tasks({ + call: async (ctx) => { + results.push( + await greeter.greet("t", ctx).catch((error: unknown) => (error as { code: string }).code), + ); + }, + }); + } + return { kind: "main" }; + }, + { eager: true }, + ), + }); + server.install(callerFacet(false)); + await sleep(10); + modules.set("greeter", greeterFacet("v2")); + await server.reload([callerFacet(true)], { remote: { "session:": ["greeter"] } }); + await until(() => results.length === 1); + assert.deepEqual(results, ["v2 t"]); + }); + + it("#5 a child that missed a commit is reconciled on reconnect, never relabeled", async () => { + const { server, worker, modules, link, connect } = setup(); + await tick(); + modules.set("greeter", greeterFacet("v2")); + // The commit frame is lost: the link dies right after the child voted. + link.closeOn = (frame) => frame.t === "commit"; + await server.reload([], { remote: { "session:": ["greeter"] } }); + await until(() => !internals(worker).reloading); + assert.notEqual(worker.epoch, server.epoch); + const next = connect(); + worker.connect(next.up); + const greeter = server.get(Greeter, {}, { at: "session:S1" }); + assert.equal(await greeter.greet("x"), "v2 x"); + assert.equal(worker.epoch, server.epoch); + }); + + it("#6 a server root that bounces through a worker and back ends normally (no leaked lease)", async () => { + const { server, worker } = setup(); + await tick(); + const Echo = remote("t.echo2", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { ping: method(Type.Null(), Type.Null()) } } }, + }); + const Bouncer = remote("t.bouncer", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { bounce: method(Type.Null(), Type.Null()) } } }, + }); + const echoFacet = (version: number): Facet => ({ + name: "echo", + setup: (api) => + api.serve( + Echo, + (_input, _principal, self) => { + self.state({ n: cell(Type.Number(), 0) }); + return { kind: "main", ping: () => null }; + }, + { state: { version } }, + ), + }); + server.install(echoFacet(1)); + // Keeps the Echo record the bounced call reaches (it acts as the worker's principal) alive, so the reload gate + // must decide about the bounced root's lease. + const echo = server.get(Echo, {}, { principal: { worker: "w" } }); + await echo.ready; + worker.serve(Bouncer, () => ({ + kind: "main", + bounce: async (_args: null, ctx: Ctx) => { + await ctx.get(Echo, {}, { at: "server" }).ping(null, ctx); + return null; + }, + })); + await server.root((ctx) => server.get(Bouncer, {}, { at: "session:S1" }).bounce(null, ctx)); + await sleep(10); + await server.reload([echoFacet(2)], { gateDeadline: 200 }); + }); +}); diff --git a/packages/chord/test/loader.test.ts b/packages/chord/test/loader.test.ts new file mode 100644 index 000000000..66b840d0f --- /dev/null +++ b/packages/chord/test/loader.test.ts @@ -0,0 +1,21 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "vitest"; +import { createHost } from "../src/index.ts"; + +describe("node loader", () => { + it("loads facet modules and reloads them with fresh copies of their relative imports", () => { + const child = spawnSync(process.execPath, [fileURLToPath(new URL("./fixtures/loader.ts", import.meta.url))], { + encoding: "utf8", + timeout: 30_000, + }); + assert.equal(child.error, undefined, child.stderr); + assert.equal(child.status, 0, `${child.stdout}\n${child.stderr}`); + }); + + it("needs a configured loader for modules", async () => { + const host = createHost({ onError: () => {} }); + await assert.rejects(host.load(["./facet.ts"]), /createHost\(\{ loader \}\)/); + }); +}); diff --git a/packages/chord/test/manager.test.ts b/packages/chord/test/manager.test.ts new file mode 100644 index 000000000..42f403c9b --- /dev/null +++ b/packages/chord/test/manager.test.ts @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import { readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, it } from "vitest"; +import { createHost, ipcLink, isChordError } from "../src/index.ts"; +import { type ControlMessage, createHostManager, type Driver, forkDriver } from "../src/manager.ts"; +import { Echo } from "./fixtures/echo.ts"; + +const entry = fileURLToPath(new URL("./fixtures/worker.ts", import.meta.url)); +const scratch = (name: string) => join(tmpdir(), `chord2-${process.pid}-${name}`); + +const alive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +const driver = (env: Record = {}, lock?: string): Driver => + forkDriver({ entry, env: { HOST_NAME: "w1", ...env }, init: { greeting: "hi" }, lock }); + +describe("host manager (real child processes)", () => { + it("starts a child host, links it, stops it gracefully", async () => { + const manager = createHostManager({ driver: driver() }); + const instance = await manager.start(); + assert.equal(manager.status, "running"); + const server = createHost({ name: "server", onError: () => {} }); + server.accept(instance.link, { principal: { worker: "w1" }, kind: "host", name: "w1" }); + const echo = server.get(Echo, {}, { at: "w1" }); + assert.equal(await echo.echo("x"), `hi x from ${instance.pid}`); + assert.equal(await manager.start(), instance); + await manager.stop(); + assert.equal(manager.status, "stopped"); + assert.deepEqual(await instance.exited, { code: 0, signal: null }); + await server.close(); + }); + + it("a child-owned kernel lock admits one live process; replace waits for the exit before the next start", async () => { + const lock = `chord2-test-${process.pid}-lock`; + const pids: number[] = []; + const base = driver({}, lock); + const watched: Driver = { + spawn() { + // Exit before replace: the previous child is gone when the next one spawns. + for (const pid of pids) assert.ok(!alive(pid), `pid ${pid} still alive`); + const instance = base.spawn(); + pids.push(instance.pid); + return instance; + }, + }; + const first = createHostManager({ driver: watched }); + const second = createHostManager({ driver: driver({}, lock) }); + const a = await first.start(); + // Both managers race for the same lock; only the kernel decides, so the second child cannot start. + await assert.rejects(second.start(), (error) => isChordError(error, "unavailable") && error.reason === "locked"); + const b = await first.replace(); + assert.notEqual(a.pid, b.pid); + await first.stop(); + await second.start(); + await second.stop(); + }); + + it("a graceful exit still takes the child's descendants with it", async () => { + const file = scratch("grandchild-graceful"); + rmSync(file, { force: true }); + const manager = createHostManager({ driver: driver({ GRANDCHILD_FILE: file }) }); + await manager.start(); + const grandchild = Number(readFileSync(file, "utf8")); + assert.ok(alive(grandchild)); + await manager.stop(); + for (let i = 0; i < 50 && alive(grandchild); i++) await new Promise((resolve) => setTimeout(resolve, 20)); + assert.ok(!alive(grandchild)); + rmSync(file, { force: true }); + }); + + it("idleFor counts from the busy → idle transition", async () => { + const listeners = new Set<(message: ControlMessage) => void>(); + const fake: Driver = { + spawn: () => ({ + pid: 0, + link: ipcLink({ connected: false, send: () => false, on: () => undefined, off: () => undefined }), + exited: new Promise(() => {}), + send: () => {}, + killTree: () => {}, + onControl: (listener) => listeners.add(listener), + }), + }; + const emit = (message: ControlMessage) => { + for (const listener of listeners) listener(message); + }; + const manager = createHostManager({ driver: fake }); + const started = manager.start(); + for (let i = 0; i < 20 && listeners.size === 0; i++) await new Promise((resolve) => setImmediate(resolve)); + emit({ type: "ready" }); + await started; + emit({ type: "status", activeCalls: 1 }); + await new Promise((resolve) => setTimeout(resolve, 120)); + assert.equal(manager.liveness.idleFor, 0); + emit({ type: "status", activeCalls: 0 }); + assert.ok(manager.liveness.idleFor < 60, `idleFor ${manager.liveness.idleFor}`); + }); + + it("kills the process tree when a child ignores shutdown past the grace period", async () => { + const file = scratch("grandchild"); + rmSync(file, { force: true }); + const manager = createHostManager({ + driver: driver({ FIXTURE_MODE: "ignore-shutdown", GRANDCHILD_FILE: file }), + grace: 300, + }); + const instance = await manager.start(); + const grandchild = Number(readFileSync(file, "utf8")); + assert.ok(alive(grandchild)); + await manager.stop(); + assert.equal((await instance.exited).signal, "SIGKILL"); + for (let i = 0; i < 50 && alive(grandchild); i++) await new Promise((resolve) => setTimeout(resolve, 20)); + assert.ok(!alive(grandchild)); + rmSync(file, { force: true }); + }); + + it("reports a fatal child as a failed start", async () => { + const manager = createHostManager({ driver: driver({ FIXTURE_MODE: "fatal" }) }); + await assert.rejects(manager.start(), /fixture refuses to start/); + assert.equal(manager.status, "stopped"); + }); + + it("stops an idle child when keepAlive says so", async () => { + const exits: unknown[] = []; + const manager = createHostManager({ + driver: driver(), + keepAlive: (liveness) => !liveness.booting && liveness.idleFor < 150, + checkInterval: 50, + onExit: (exit) => exits.push(exit), + }); + await manager.start(); + for (let i = 0; i < 100 && manager.status !== "stopped"; i++) + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(manager.status, "stopped"); + assert.equal(exits.length, 1); + }); +}); diff --git a/packages/chord/test/readiness.test.ts b/packages/chord/test/readiness.test.ts new file mode 100644 index 000000000..8c7d7d0b4 --- /dev/null +++ b/packages/chord/test/readiness.test.ts @@ -0,0 +1,258 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { + createHost, + defineRegistry, + type FacetApi, + forbidden, + isChordError, + local, + memoryLinkPair, + remote, + resource, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const unexpectedError = (error: unknown) => assert.fail(String(error)); +const Eager = local("ready.eager", { version: 1, input: Type.Object({}) }); +const Lazy = local("ready.lazy", { version: 1, input: Type.Object({}) }); + +// Request #11 (docs/requests.md): startup must not report ready while an eager object has failed. +describe("host: eager readiness", () => { + for (const operation of ["install", "load"] as const) { + it(`${operation} awaits resource activation and registration, leaving lazy objects lazy`, async () => { + const host = createHost({ onError: unexpectedError }); + const entries = defineRegistry("ready.entries"); + let opened = () => {}; + let lazyRuns = 0; + let settled = false; + const facet = { + name: "ready", + setup({ serve }: FacetApi) { + serve( + Eager, + (_input, _principal, self) => { + self.resources({ + r: resource({ + rev: 1, + open: () => + new Promise((resolve) => { + opened = resolve; + }), + }), + }); + self.register(entries, { eager: "live" }); + return {}; + }, + { eager: true }, + ); + serve(Lazy, () => { + lazyRuns++; + return {}; + }); + }, + }; + try { + const loading = operation === "install" ? host.install(facet) : host.load([facet]); + void loading.then(() => { + settled = true; + }); + await tick(); + assert.equal(settled, false); + assert.equal(host.registry(entries).get("eager"), undefined); + opened(); + await loading; + assert.equal(host.registry(entries).get("eager"), "live"); + assert.equal(lazyRuns, 0); + } finally { + await host.close(); + } + }); + + for (const failure of ["factory", "dependency", "resource"] as const) { + it(`${operation} rejects eager ${failure} failure with the activation error`, async () => { + const errors: unknown[] = []; + const host = createHost({ onError: (error) => errors.push(error) }); + const cause = forbidden(`${failure} failed`); + const facet = { + name: "bad", + setup({ serve }: FacetApi) { + serve(Lazy, () => { + throw cause; + }); + serve( + Eager, + (_input, _principal, self) => { + if (failure === "factory") throw cause; + if (failure === "dependency") self.get(Lazy, {}); + if (failure === "resource") + self.resources({ + r: resource({ + rev: 1, + open: () => { + throw cause; + }, + }), + }); + return {}; + }, + { eager: true }, + ); + }, + }; + try { + await assert.rejects( + operation === "install" ? host.install(facet) : host.load([facet]), + (error) => error === cause, + ); + await assert.rejects(host.ready(), (error) => error === cause); + assert.deepEqual(errors, [cause]); + } finally { + await host.close(); + } + }); + } + } + + it("load installs every facet before awaiting eager consumers", async () => { + const host = createHost({ onError: unexpectedError }); + try { + await host.load([ + { + name: "consumer", + setup: ({ serve }) => + serve( + Eager, + (_input, _principal, self) => { + self.get(Lazy, {}); + return {}; + }, + { eager: true }, + ), + }, + { name: "provider", setup: ({ serve }) => serve(Lazy, () => ({})) }, + ]); + assert.equal(host.get(Eager, {}).status, "live"); + } finally { + await host.close(); + } + }); + + it("missing eager dependencies reject load instead of only reaching onError", async () => { + const host = createHost({ onError: () => {} }); + try { + await assert.rejects( + host.load([ + { + name: "missing", + setup: ({ serve }) => + serve( + Eager, + (_i, _p, self) => { + self.get(Lazy, {}); + return {}; + }, + { eager: true }, + ), + }, + ]), + (error) => isChordError(error, "unavailable") && error.reason === "unknown_type", + ); + } finally { + await host.close(); + } + }); + + it("ready includes direct eager serves and their configured inputs, but not later serves", async () => { + const host = createHost({ onError: () => {} }); + const WithInput = local("ready.input", { version: 1, input: Type.Object({ id: Type.String() }) }); + let input: unknown; + try { + host.serve( + WithInput, + (value) => { + input = value; + return {}; + }, + { eager: { input: { id: "bootstrap" } } }, + ); + const first = host.ready(); + host.serve( + Eager, + () => { + throw forbidden("later"); + }, + { eager: true }, + ); + await first; + assert.deepEqual(input, { id: "bootstrap" }); + await assert.rejects(host.ready(), /later/); + } finally { + await host.close(); + } + }); + + for (const operation of ["install", "load"] as const) { + it(`${operation} can defer eager readiness until the parent admits the worker`, { timeout: 2000 }, async () => { + const server = createHost({ name: "server", onError: unexpectedError }); + const [parent, upstream] = memoryLinkPair(); + const worker = createHost({ name: "worker", upstream, onError: unexpectedError }); + const Status = remote("ready.status", { version: 1, input: Type.Object({}), variants: { default: {} } }); + server.serve(Status, () => ({ kind: "default" })); + let bootstrap = false; + let reported = false; + const facet = { + name: "worker", + setup({ serve }: FacetApi) { + serve( + Lazy, + () => { + bootstrap = true; + return {}; + }, + { eager: true }, + ); + serve( + Eager, + (_input, _principal, self) => { + const status = self.get(Status, {}, { at: "server" }); + self.resources({ + r: resource({ + rev: 1, + deps: [status], + open: () => { + reported = true; + }, + }), + }); + return {}; + }, + { eager: true }, + ); + }, + }; + try { + await (operation === "install" + ? worker.install(facet, { awaitEager: false }) + : worker.load([facet], { awaitEager: false })); + await worker.get(Lazy, {}).ready; + assert.equal(bootstrap, true); + let settled = false; + const ready = worker.ready().then(() => { + settled = true; + }); + await tick(); + assert.equal(settled, false); + assert.equal(reported, false); + // Control-ready announcement allows the parent to accept the queued upstream holds. + server.accept(parent, { kind: "host", name: "worker", principal: { host: "worker" } }); + await ready; + assert.equal(reported, true); + } finally { + await worker.close(); + await server.close(); + } + }); + } +}); diff --git a/packages/chord/test/reload.test.ts b/packages/chord/test/reload.test.ts new file mode 100644 index 000000000..04c45618e --- /dev/null +++ b/packages/chord/test/reload.test.ts @@ -0,0 +1,568 @@ +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { + type Ctx, + cell, + createHost, + defineRegistry, + type Facet, + isChordError, + listener, + local, + method, + remote, + resource, + type Self, + type ServeOptions, + type StateEvent, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const quiet = () => createHost({ onError: () => {} }); + +function latch() { + let open = () => {}; + const promise = new Promise((resolve) => { + open = resolve; + }); + return { promise, open }; +} + +const Greeter = local("t.greeter", { + version: 1, + input: Type.Object({ id: Type.Optional(Type.String({ default: "a" })) }), + members: { greet: method(Type.String(), Type.String()), count: Type.Unknown() }, +}); + +const CountV1 = Type.Object({ n: Type.Number() }); + +interface GreeterSpec { + readonly greeting: string; + readonly schema?: typeof CountV1 | ReturnType; + readonly initial?: object; + readonly bump?: (draft: Record) => void; + readonly options?: ServeOptions; + readonly extra?: (self: Self, input: { id?: string }) => void; +} + +function greeterFacet(spec: GreeterSpec): Facet { + return { + name: "greeter", + setup: (api) => + api.serve( + Greeter, + (input, _principal, self) => { + spec.extra?.(self, input); + const state = self.state({ count: cell(spec.schema ?? CountV1, (spec.initial ?? { n: 0 }) as never) }); + return { + count: state.count, + greet: (name: string, ctx: Ctx) => { + state.count.change(ctx, (draft) => { + const bump = + spec.bump ?? + ((d: Record) => { + d.n = (d.n ?? 0) + 1; + }); + bump(draft as Record); + }); + return `${spec.greeting} ${name}`; + }, + }; + }, + spec.options, + ), + }; +} + +async function until(check: () => Promise, attempts = 200): Promise { + for (let attempt = 0; attempt < attempts; attempt++) { + if (await check()) return; + await tick(); + } + throw new Error("condition not reached"); +} + +describe("reload", () => { + it("compatible reload: no gate, pinned roots keep old code, state preserved, previous retires", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const handle = host.get(Greeter, {}); + assert.equal(await handle.greet("a"), "hello a"); + const go = latch(); + const pinned = host.root(async (ctx) => { + const first = await handle.greet("p", ctx); + await go.promise; + return [first, await handle.greet("p", ctx)]; + }); + await tick(); + const shapes: number[] = []; + handle.onShape((shape) => shapes.push(shape.epoch)); + const report = await host.reload([greeterFacet({ greeting: "hi" })]); + assert.deepEqual(report.gated, []); + assert.equal(report.prepared.length, 1); + assert.equal(await handle.greet("b"), "hi b"); + assert.deepEqual(shapes, [report.epoch]); + assert.equal(host.previousEpoch, report.previous); + go.open(); + assert.deepEqual(await pinned, ["hello p", "hello p"]); + assert.equal(host.previousEpoch, undefined); + assert.deepEqual(handle.count.value, { n: 4 }); + }); + + it("prepare runs in recording mode: writes throw and a failing factory leaves everything unchanged", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const a = host.get(Greeter, { id: "a" }); + const b = host.get(Greeter, { id: "b" }); + await Promise.all([a.ready, b.ready]); + let captured: Ctx | undefined; + await host.root((ctx) => { + captured = ctx; + }); + + const writing = greeterFacet({ + greeting: "hi", + extra: (self) => { + const state = self.state({ probe: cell(Type.Object({}), {}) }); + state.probe.replace(captured!, {}); + }, + }); + await assert.rejects(host.reload([writing]), /read_only/); + + const log: string[] = []; + const failing = greeterFacet({ + greeting: "hi", + extra: (self, input) => { + self.resources({ + extra: resource({ + rev: 1, + open: () => void log.push(`open ${input.id}`), + close: () => void log.push(`close ${input.id}`), + }), + }); + if (input.id === "b") throw new Error("b fails"); + }, + }); + const epoch = host.epoch; + await assert.rejects( + host.reload([failing]), + // A plain error from the new factory surfaces as `unavailable`; the message says the reload aborted. + (error) => isChordError(error, "unavailable") && /reload aborted.*b fails/.test(error.message), + ); + assert.equal(host.epoch, epoch); + assert.deepEqual(log, ["open a", "close a"]); + assert.equal(await a.greet("x"), "hello x"); + }); + + it("refuses a changed cell schema without a state-version bump", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + await host.get(Greeter, {}).ready; + const changed = greeterFacet({ + greeting: "hi", + schema: Type.Object({ total: Type.Number() }), + initial: { total: 0 }, + }); + await assert.rejects( + host.reload([changed]), + (error) => + isChordError(error, "invalid") && + error.reason === "state" && + /without a state-version bump/.test(error.message), + ); + }); + + it("state bump: gate, migrate(old, fromVersion) into the new schema, reset to watchers, old roots upgrade_conflict", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const handle = host.get(Greeter, {}); + await handle.greet("a"); + await handle.greet("a"); + const events: StateEvent[] = []; + handle.count.events((event) => events.push(event)); + const go = latch(); + // Started before the reload, never touched the record: rejected after the commit. + const late = host.root(async (ctx) => { + await go.promise; + return handle.greet("late", ctx); + }); + const froms: number[] = []; + const v2 = greeterFacet({ + greeting: "hi", + schema: Type.Object({ total: Type.Number() }), + initial: { total: 0 }, + bump: (draft) => { + draft.total = (draft.total ?? 0) + 1; + }, + options: { + state: { + version: 2, + migrate: { + count: (old, from) => { + froms.push(from); + return { total: (old as { n: number }).n }; + }, + }, + }, + }, + }); + const report = await host.reload([v2]); + assert.equal(report.gated.length, 1); + assert.deepEqual(froms, [1]); + assert.deepEqual(events.at(-1), { type: "reset", value: { total: 2 } }); + assert.equal(await handle.greet("b"), "hi b"); + assert.deepEqual(handle.count.value, { total: 3 }); + go.open(); + await assert.rejects(late, (error) => isChordError(error, "upgrade_conflict") && error.retryable); + }); + + it("gate: late old roots get upgrade_conflict, grandfathered roots re-enter, then commit", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const handle = host.get(Greeter, {}); + await handle.ready; + const go = latch(); + const grandfathered = host.root(async (ctx) => { + await handle.greet("g", ctx); + await go.promise; + return handle.greet("g", ctx); + }); + await tick(); + const reload = host.reload([greeterFacet({ greeting: "hi", options: { state: { version: 2 } } })]); + await until(async () => { + try { + await handle.greet("probe"); + return false; + } catch (error) { + return isChordError(error, "upgrade_conflict"); + } + }); + go.open(); + assert.equal(await grandfathered, "hello g"); + const report = await reload; + assert.equal(report.gated.length, 1); + assert.equal(await handle.greet("n"), "hi n"); + }); + + it("gate deadline: reload aborts, reports blocking roots, nothing changes", async () => { + const host = quiet(); + const log: string[] = []; + const facet = (greeting: string, version: number): Facet => ({ + name: "greeter", + setup: (api) => + api.serve( + Greeter, + (_input, _principal, self) => { + self.tasks({ + loop: async (ctx) => { + log.push(`start ${greeting}`); + await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); + log.push(`stop ${greeting}`); + }, + }); + const state = self.state({ count: cell(Type.Number(), 0) }); + return { count: state.count, greet: (name: string) => `${greeting} ${name}` }; + }, + { state: { version } }, + ), + }); + host.install(facet("hello", 1)); + const handle = host.get(Greeter, {}); + await handle.ready; + const go = latch(); + const blocker = host.root(async (ctx) => { + await handle.greet("b", ctx); + await go.promise; + }); + await tick(); + await assert.rejects(host.reload([facet("hi", 2)], { gateDeadline: 30 }), (error) => { + assert.ok(isChordError(error, "aborted") && error.reason === "gate_deadline"); + assert.match(error.message, /blocking roots: host#\w+:\d+/); + return true; + }); + assert.equal(await handle.greet("x"), "hello x"); + await tick(); + assert.deepEqual(log, ["start hello", "stop hello", "start hello"]); + go.open(); + await blocker; + }); + + it("exclusive resources are replaced at the gate; non-exclusive ones at prepare and closed at retirement", async () => { + const host = quiet(); + const log: string[] = []; + const facet = (rev: number): Facet => ({ + name: "greeter", + setup: (api) => + api.serve(Greeter, (_input, _principal, self) => { + const res = self.resources({ + lock: resource({ + rev, + exclusive: true, + open: () => { + log.push(`open lock ${rev}`); + return rev; + }, + close: () => void log.push(`close lock ${rev}`), + }), + cache: resource({ + rev, + open: () => void log.push(`open cache ${rev}`), + close: () => void log.push(`close cache ${rev}`), + }), + }); + const state = self.state({ count: cell(Type.Number(), 0) }); + return { count: state.count, greet: (name: string) => `${res.lock.value} ${name}` }; + }), + }); + host.install(facet(1)); + const handle = host.get(Greeter, {}); + assert.equal(await handle.greet("x"), "1 x"); + log.length = 0; + const report = await host.reload([facet(2)]); + assert.equal(report.gated.length, 1); + assert.equal(await handle.greet("x"), "2 x"); + await tick(); + assert.deepEqual(log, ["open cache 2", "close lock 1", "open lock 2", "close cache 1"]); + }); + + it("two-version rule: the next reload retires the previous epoch; its roots get superseded", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "v1" })); + const handle = host.get(Greeter, {}); + await handle.ready; + const first = host.epoch; + const go = latch(); + const pinned = host.root(async (ctx) => { + await handle.greet("p", ctx); + await go.promise; + return handle.greet("p", ctx); + }); + await tick(); + await host.reload([greeterFacet({ greeting: "v2" })]); + assert.equal(host.previousEpoch, first); + await host.reload([greeterFacet({ greeting: "v3" })], { retireDeadline: 20 }); + go.open(); + await assert.rejects(pinned, (error) => isChordError(error, "superseded") && error.retryable); + await assert.rejects( + host.root(() => 1, { epoch: first }), + (error) => isChordError(error, "superseded"), + ); + assert.equal(await handle.greet("n"), "v3 n"); + }); + + it("checks contracts at reload: forgotten bumps, live consumers, accepts adapters", async () => { + const DocV1 = remote("t.rdoc", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { read: method(Type.Null(), Type.Object({ text: Type.String() })) } } }, + }); + const DocBroken = remote("t.rdoc", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { read: method(Type.Null(), Type.Object({ text: Type.Number() })) } } }, + }); + const DocV2 = remote("t.rdoc", { + version: 2, + input: Type.Object({}), + variants: { main: { methods: { fetch: method(Type.Null(), Type.Object({ body: Type.String() })) } } }, + }); + const host = quiet(); + host.install({ + name: "doc", + setup: (api) => api.serve(DocV1, () => ({ kind: "main", read: () => ({ text: "v1" }) })), + }); + const consumer = host.get(DocV1, {}); + assert.deepEqual(await consumer.read(null), { text: "v1" }); + + const broken: Facet = { + name: "doc", + setup: (api) => api.serve(DocBroken, () => ({ kind: "main", read: () => ({ text: 1 }) })), + }; + await assert.rejects(host.reload([broken]), /without a version bump/); + + const fetch = () => ({ body: "v2" }); + const v2: Facet = { name: "doc", setup: (api) => api.serve(DocV2, () => ({ kind: "main", fetch })) }; + await assert.rejects(host.reload([v2]), (error) => isChordError(error, "contract_mismatch")); + + const adapted: Facet = { + name: "doc", + setup: (api) => + api.serve(DocV2, () => ({ kind: "main", fetch }), { + accepts: { + 1: (current) => ({ + kind: "main", + read: () => ({ text: `${(current.fetch as typeof fetch)().body} via adapter` }), + }), + }, + }), + }; + await host.reload([adapted]); + assert.deepEqual(await consumer.read(null), { text: "v2 via adapter" }); + const modern = host.get(DocV2, {}); + assert.deepEqual(await modern.fetch(null), { body: "v2" }); + }); + + it("rebinds listeners to the new code and restarts tasks", async () => { + const host = quiet(); + const listeners = new Set<(event: number) => void>(); + const events = { + subscribe: (fn: (event: number) => void) => { + listeners.add(fn); + return () => listeners.delete(fn); + }, + }; + const emit = (value: number) => { + for (const fn of listeners) fn(value); + }; + const log: string[] = []; + const Sum = local("t.sum", { + version: 1, + input: Type.Object({}), + members: { sum: Type.Object({ v: Type.Number() }) }, + }); + const facet = (factor: number): Facet => ({ + name: "sum", + setup: (api) => + api.serve(Sum, (_input, _principal, self) => { + const state = self.state({ sum: cell(Type.Object({ v: Type.Number() }), { v: 0 }) }); + self.listen({ + add: listener( + events, + (source) => source, + (value, ctx) => + state.sum.change(ctx, (draft) => { + draft.v += value * factor; + }), + ), + }); + self.tasks({ + run: async (ctx) => { + log.push(`start ${factor}`); + await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); + log.push(`stop ${factor}`); + }, + }); + return { sum: state.sum }; + }), + }); + host.install(facet(1)); + const handle = host.get(Sum, {}); + await handle.ready; + emit(1); + await tick(); + await host.reload([facet(10)]); + assert.equal(listeners.size, 1); + emit(1); + await tick(); + assert.deepEqual(handle.sum.value, { v: 11 }); + assert.deepEqual(log, ["start 1", "start 10", "stop 1"]); + }); + + it("keeps registry tables per epoch", async () => { + const host = quiet(); + const Routes = defineRegistry("t.reload-routes"); + const Router = local("t.router", { version: 1, input: Type.Object({}) }); + const facet = (value: string): Facet => ({ + name: "router", + setup: (api) => + api.serve( + Router, + (_input, _principal, self) => { + self.register(Routes, { "GET /": value }); + return {}; + }, + { eager: true }, + ), + }); + host.install(facet("one")); + await tick(); + await tick(); + const seen: [string | undefined, number][] = []; + host.registry(Routes).subscribe((table, epoch) => seen.push([table.get("GET /"), epoch])); + const go = latch(); + const pinned = host.root(async (ctx) => { + await go.promise; + return host.registry(Routes).get("GET /", ctx); + }); + const report = await host.reload([facet("two")]); + assert.equal(host.registry(Routes).get("GET /"), "two"); + go.open(); + assert.equal(await pinned, "one"); + assert.deepEqual(seen, [ + ["one", report.previous], + ["two", report.epoch], + ]); + }); + + it("a reload can add a type together with a consumer that holds it", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const handle = host.get(Greeter, {}); + await handle.ready; + const Clock = local("t.clock", { + version: 1, + input: Type.Object({}), + members: { now: method(Type.Null(), Type.Number()) }, + }); + const clockFacet: Facet = { name: "clock", setup: (api) => api.serve(Clock, () => ({ now: () => 42 })) }; + const consumer = (fail: boolean): Facet => ({ + name: "greeter", + setup: (api) => + api.serve(Greeter, (_input, _principal, self) => { + const clock = self.get(Clock, {}); + const state = self.state({ count: cell(CountV1, { n: 0 }) }); + if (fail) throw new Error("consumer fails"); + return { + count: state.count, + greet: async (name: string, ctx: Ctx) => `${await clock.now(null, ctx)} ${name}`, + }; + }), + }); + const epoch = host.epoch; + await assert.rejects(host.reload([clockFacet, consumer(true)]), /consumer fails/); + assert.equal(host.epoch, epoch); + assert.equal(await handle.greet("x"), "hello x"); + await host.reload([clockFacet, consumer(false)]); + assert.equal(await handle.greet("x"), "42 x"); + }); + + it("the gate rejects new records of a changed type; after commit they get the new code", async () => { + const host = quiet(); + host.install(greeterFacet({ greeting: "hello" })); + const handle = host.get(Greeter, {}); + await handle.ready; + const go = latch(); + const grandfathered = host.root(async (ctx) => { + await handle.greet("g", ctx); + await go.promise; + }); + const later = latch(); + const pinned = host.root(async (ctx) => { + await later.promise; + return handle.greet("old", ctx); + }); + await tick(); + const reload = host.reload([greeterFacet({ greeting: "hi", options: { state: { version: 2 } } })]); + await until(async () => { + try { + await handle.greet("probe"); + return false; + } catch (error) { + return isChordError(error, "upgrade_conflict"); + } + }); + assert.throws( + () => host.get(Greeter, { id: "late" }), + (error) => isChordError(error, "upgrade_conflict") && error.retryable, + ); + go.open(); + await grandfathered; + await reload; + assert.equal(await host.get(Greeter, { id: "late" }).greet("l"), "hi l"); + // The previous epoch is still retained (pinned root), but the gated record dropped its old behavior. + assert.notEqual(host.previousEpoch, undefined); + later.open(); + await assert.rejects(pinned, (error) => isChordError(error, "upgrade_conflict")); + }); +}); diff --git a/packages/chord/test/state-delivery.test.ts b/packages/chord/test/replicated-state-delivery.test.ts similarity index 70% rename from packages/chord/test/state-delivery.test.ts rename to packages/chord/test/replicated-state-delivery.test.ts index 5c1efbdec..18579cbfb 100644 --- a/packages/chord/test/state-delivery.test.ts +++ b/packages/chord/test/replicated-state-delivery.test.ts @@ -7,11 +7,9 @@ import { type ReplicatedStateSourceFrame, replicatedState, } from "../src/index.ts"; -import { ReplicatedStateReplica } from "../src/services/state.ts"; -import { getReplicatedStateInternals } from "../src/services/state-internals.ts"; type Value = { value: number }; -type Kind = "mutable" | "attached" | "replica"; +type Kind = "mutable" | "attached"; function deferred() { let resolve!: () => void; @@ -30,7 +28,7 @@ function fixture(kind: Kind, onError: (error: Error) => void = () => {}) { const mutable = replicatedState({ value: 0 }); state = mutable; publish = (value, context) => mutable.replace(context, value); - } else if (kind === "attached") { + } else { let receive!: (frame: ReplicatedStateSourceFrame) => void; state = replicatedState( { @@ -46,24 +44,17 @@ function fixture(kind: Kind, onError: (error: Error) => void = () => {}) { ); publish = (value, context) => receive({ value, cursor: value.value, ops: [["s", ["value"], value.value]], context }); - } else { - const replica = new ReplicatedStateReplica(onError); - replica.hydrate(0, [["r", { value: 0 }]], BACKGROUND_CONTEXT); - state = replica; - publish = (value, context) => replica.update(value.value, [["s", ["value"], value.value]], context); } return { state, publish }; } -describe.each(["mutable", "attached", "replica"])("%s public delivery", (kind) => { - test("awaits hydration and each update independently of other subscribers and exact listeners", async () => { +describe.each(["mutable", "attached"])("%s public delivery", (kind) => { + test("awaits hydration and each update independently of other subscribers", async () => { const { state, publish } = fixture(kind); const hydration = deferred(); const update = deferred(); const events: string[] = []; - const exact: number[] = []; const fast: number[] = []; - getReplicatedStateInternals(state)?.subscribe((_ops, sequence) => exact.push(sequence)); state.subscribe(async (value) => { events.push(`start:${value.value}`); if (value.value === 0) await hydration.promise; @@ -76,34 +67,27 @@ describe.each(["mutable", "attached", "replica"])("%s public delivery", (k expect(events).toEqual(["start:0"]); expect(fast).toEqual([0, 1, 2]); expect(state.value).toEqual({ value: 2 }); - if (kind !== "replica") expect(exact).toEqual([1, 2]); hydration.resolve(); await vi.waitFor(() => expect(events).toEqual(["start:0", "end:0", "start:1"])); update.resolve(); await vi.waitFor(() => expect(events).toEqual(["start:0", "end:0", "start:1", "end:1", "start:2", "end:2"])); }); - test.each([100, 101, 102, 201, 202])( - "bounds %i pending deliveries without changing exact publication", - async (count) => { - const { state, publish } = fixture(kind); - const hydration = deferred(); - const received: number[] = []; - const exact: number[] = []; - getReplicatedStateInternals(state)?.subscribe((_ops, sequence) => exact.push(sequence)); - state.subscribe((value) => { - received.push(value.value); - if (value.value === 0) return hydration.promise; - }); - for (let value = 1; value <= count; value += 1) publish({ value }, BACKGROUND_CONTEXT); - expect(received).toEqual([0]); - if (kind !== "replica") expect(exact).toEqual(Array.from({ length: count }, (_, index) => index + 1)); - hydration.resolve(); - await hydration.promise; - const first = Math.floor((count - 1) / 100) * 100 + 1; - expect(received).toEqual([0, ...Array.from({ length: count - first + 1 }, (_, index) => first + index)]); - }, - ); + test.each([100, 101, 102, 201, 202])("bounds %i pending deliveries", async (count) => { + const { state, publish } = fixture(kind); + const hydration = deferred(); + const received: number[] = []; + state.subscribe((value) => { + received.push(value.value); + if (value.value === 0) return hydration.promise; + }); + for (let value = 1; value <= count; value += 1) publish({ value }, BACKGROUND_CONTEXT); + expect(received).toEqual([0]); + hydration.resolve(); + await hydration.promise; + const first = Math.floor((count - 1) / 100) * 100 + 1; + expect(received).toEqual([0, ...Array.from({ length: count - first + 1 }, (_, index) => first + index)]); + }); test("excludes a running update from overflow and retains exact value/context/delivery", async () => { const { state, publish } = fixture(kind); @@ -184,7 +168,8 @@ describe.each(["mutable", "attached", "replica"])("%s public delivery", (k }); }); -describe.each(["attached", "replica"])("%s listener errors", (kind) => { +describe("attached listener errors", () => { + const kind: Kind = "attached"; test("isolates a synchronous hydration failure without removing the subscription", () => { const errors: Error[] = []; const { state, publish } = fixture(kind, (error) => errors.push(error)); @@ -261,40 +246,3 @@ test("mutable state reports rejected callbacks through its default error reporte spy.mockRestore(); } }); - -test("replica disconnect drops obsolete pending work but waits for the running callback before rehydration", async () => { - const replica = new ReplicatedStateReplica(() => {}); - const gate = deferred(); - const received: ReplicatedStateDelivery[] = []; - replica.subscribe((_value, _context, delivery) => { - received.push(delivery); - if (delivery.sequence === 0) return gate.promise; - }); - replica.hydrate(0, [["r", { value: 0 }]], BACKGROUND_CONTEXT); - replica.update(1, [["s", ["value"], 1]], BACKGROUND_CONTEXT); - replica.clear(); - replica.hydrate(50, [["r", { value: 50 }]], BACKGROUND_CONTEXT); - replica.update(51, [["s", ["value"], 51]], BACKGROUND_CONTEXT); - expect(received).toEqual([{ kind: "hydrate", sequence: 0 }]); - gate.resolve(); - await gate.promise; - expect(received).toEqual([ - { kind: "hydrate", sequence: 0 }, - { kind: "hydrate", sequence: 50 }, - { kind: "update", sequence: 51 }, - ]); -}); - -test("cold replicas hydrate all listeners before their reentrant updates", () => { - const replica = new ReplicatedStateReplica(() => {}); - const second: ReplicatedStateDelivery[] = []; - replica.subscribe((_value, _context, delivery) => { - if (delivery.kind === "hydrate") replica.update(1, [["s", ["value"], 1]], BACKGROUND_CONTEXT); - }); - replica.subscribe((_value, _context, delivery) => second.push(delivery)); - replica.hydrate(0, [["r", { value: 0 }]], BACKGROUND_CONTEXT); - expect(second).toEqual([ - { kind: "hydrate", sequence: 0 }, - { kind: "update", sequence: 1 }, - ]); -}); diff --git a/packages/chord/test/replicated-state.test.ts b/packages/chord/test/replicated-state.test.ts new file mode 100644 index 000000000..0ef38e19b --- /dev/null +++ b/packages/chord/test/replicated-state.test.ts @@ -0,0 +1,351 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; +import type { Draft, Op } from "../src/delta/index.ts"; +import { + type Context, + type ReplicatedStateSource, + type ReplicatedStateSourceAttachment, + type ReplicatedStateSourceFrame, + replicatedState, +} from "../src/index.ts"; + +describe("transactional replicated state", () => { + it("publishes one immutable structurally shared revision", () => { + const initial = { changed: { value: 1 }, retained: { value: 2 } }; + const state = replicatedState(initial); + const previous = state.value; + const deliveries: number[] = []; + state.subscribe((_value, _context, delivery) => deliveries.push(delivery.sequence)); + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.changed.value = 3; + draft.changed.value = 4; + }); + expect(state.value).toEqual({ changed: { value: 4 }, retained: { value: 2 } }); + expect(state.value).not.toBe(previous); + expect(state.value.changed).not.toBe(previous.changed); + expect(state.value.retained).toBe(previous.retained); + expect(Object.isFrozen(state.value)).toBe(false); + expect(deliveries).toEqual([0, 1]); + }); + + it("rolls back callback failures and revokes escaped drafts", () => { + const state = replicatedState({ nested: { value: 1 } }); + const previous = state.value; + let escaped: Draft<{ value: number }> | undefined; + expect(() => + state.change(BACKGROUND_CONTEXT, (draft) => { + escaped = draft.nested; + draft.nested.value = 2; + throw new Error("stop"); + }), + ).toThrow("stop"); + expect(state.value).toBe(previous); + expect(() => escaped?.value).toThrow(TypeError); + }); + + it("rejects PromiseLike callbacks and aborts their draft", () => { + const state = replicatedState({ value: 0 }); + const previous = state.value; + expect(() => + state.change(BACKGROUND_CONTEXT, ((draft: Draft<{ value: number }>) => { + draft.value = 1; + return Promise.resolve(); + }) as (draft: Draft<{ value: number }>) => void), + ).toThrow(/synchronous/); + expect(state.value).toBe(previous); + }); + + it("rejects nested changes and replacements without losing the outer rollback", () => { + const state = replicatedState({ left: 0, right: 0 }); + expect(() => + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.left = 1; + state.change(BACKGROUND_CONTEXT, (nested) => { + nested.right = 2; + }); + }), + ).toThrow(/reentrantly/); + expect(state.value).toEqual({ left: 0, right: 0 }); + expect(() => + state.change(BACKGROUND_CONTEXT, () => { + state.replace(BACKGROUND_CONTEXT, { left: 1, right: 2 }); + }), + ).toThrow(/change callback/); + expect(state.value).toEqual({ left: 0, right: 0 }); + }); + + it("queues listener-triggered changes in sequence order", () => { + const state = replicatedState({ value: 0 }); + const deliveries: Array<{ value: number; sequence: number }> = []; + const lateDeliveries: Array<{ kind: string; sequence: number }> = []; + let nested = false; + state.subscribe((_value, _context, delivery) => { + if (delivery.kind !== "update" || nested) return; + nested = true; + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.value = 2; + }); + state.subscribe((_value, _context, delivery) => lateDeliveries.push(delivery)); + }); + state.subscribe((value, _context, delivery) => { + if (delivery.kind === "update") deliveries.push({ value: value.value, sequence: delivery.sequence }); + }); + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.value = 1; + }); + expect(deliveries).toEqual([ + { value: 1, sequence: 1 }, + { value: 2, sequence: 2 }, + ]); + expect(lateDeliveries).toEqual([{ kind: "hydrate", sequence: 2 }]); + }); + + it("isolates listener failures after committing the revision", () => { + const reports: (() => void)[] = []; + const spy = vi.spyOn(globalThis, "queueMicrotask").mockImplementation((report) => { + reports.push(report); + }); + try { + const state = replicatedState({ value: 0 }); + const received: number[] = []; + state.subscribe((_value, _context, delivery) => { + if (delivery.kind === "update") throw new Error("listener failed"); + }); + state.subscribe((_value, _context, delivery) => received.push(delivery.sequence)); + expect(() => + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.value = 1; + }), + ).not.toThrow(); + expect(state.value).toEqual({ value: 1 }); + expect(received).toEqual([0, 1]); + expect(reports).toHaveLength(1); + expect(reports[0]).toThrow("listener failed"); + } finally { + spy.mockRestore(); + } + }); + + it("copies assigned values by value", () => { + const external = { value: 1 }; + const state = replicatedState<{ left: { value: number } | null; right: { value: number } | null }>({ + left: null, + right: null, + }); + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.left = external; + draft.right = external; + draft.left.value = 2; + }); + expect(external.value).toBe(1); + expect(state.value).toEqual({ left: { value: 2 }, right: { value: 1 } }); + expect(state.value.left).not.toBe(state.value.right); + }); + + it("takes immutable ownership of alias-free replacements", () => { + const state = replicatedState({ left: { value: 1 }, right: { value: 2 } }); + const replacement = { left: { ...state.value.left }, right: { ...state.value.left } }; + state.replace(BACKGROUND_CONTEXT, replacement); + expect(state.value).toBe(replacement); + expect(state.value.left).not.toBe(state.value.right); + state.change(BACKGROUND_CONTEXT, (draft) => { + draft.left.value = 9; + }); + expect(state.value).toEqual({ left: { value: 9 }, right: { value: 1 } }); + }); + + it("replaces atomically and ignores deeply equal replacements", () => { + const state = replicatedState({ value: { nested: 1 }, retained: { nested: 2 } }); + const previous = state.value; + state.replace(BACKGROUND_CONTEXT, { value: { nested: 1 }, retained: { nested: 2 } }); + expect(state.value).toBe(previous); + state.replace(BACKGROUND_CONTEXT, { ...state.value, value: { nested: 2 } }); + expect(state.value).toEqual({ value: { nested: 2 }, retained: { nested: 2 } }); + expect(state.value.retained).toBe(previous.retained); + }); +}); + +type SourceValue = Readonly<{ value: number }>; + +class TestSource implements ReplicatedStateSource { + readonly attachments = new Set>(); + onAttach: (() => void) | undefined; + #value: T; + #cursor: number; + + constructor(value: T, cursor = 0) { + this.#value = value; + this.#cursor = cursor; + } + + attach(): ReplicatedStateSourceAttachment { + const attachment = new TestSourceAttachment(Object.freeze({ value: this.#value, cursor: this.#cursor }), () => + this.attachments.delete(attachment), + ); + this.attachments.add(attachment); + this.onAttach?.(); + return attachment; + } + + commit(value: T, ops: readonly Op[], context: Context = BACKGROUND_CONTEXT, cursor = this.#cursor + 1): void { + this.#value = value; + this.#cursor = cursor; + const frame = Object.freeze({ cursor, value, ops, context }); + for (const attachment of [...this.attachments]) attachment.publish(frame); + } +} + +class TestSourceAttachment implements ReplicatedStateSourceAttachment { + readonly snapshot: { readonly value: T; readonly cursor: number }; + readonly #onDispose: () => void; + readonly #buffer: ReplicatedStateSourceFrame[] = []; + #listener: ((frame: ReplicatedStateSourceFrame) => void) | undefined; + #activated = false; + #disposed = false; + + constructor(snapshot: { readonly value: T; readonly cursor: number }, onDispose: () => void) { + this.snapshot = snapshot; + this.#onDispose = onDispose; + } + + activate(listener: (frame: ReplicatedStateSourceFrame) => void): void { + if (this.#activated) throw new Error("attachment is already active"); + this.#activated = true; + this.#listener = listener; + for (const frame of this.#buffer.splice(0)) listener(frame); + } + + publish(frame: ReplicatedStateSourceFrame): void { + if (this.#disposed) return; + if (this.#listener === undefined) this.#buffer.push(frame); + else this.#listener(frame); + } + + dispose(): void { + if (this.#disposed) return; + this.#disposed = true; + this.#buffer.length = 0; + this.#onDispose(); + } +} + +describe("authoritative replicated state sources", () => { + it("captures before activation and drains queued commits in order", () => { + const initial = Object.freeze({ value: 0 }); + const first = Object.freeze({ value: 1 }); + const second = Object.freeze({ value: 2 }); + const firstOps = Object.freeze([["s", ["value"], 1] as const]); + const secondOps = Object.freeze([["s", ["value"], 2] as const]); + const source = new TestSource(initial, 10); + source.onAttach = () => { + source.commit(first, firstOps); + source.commit(second, secondOps); + }; + + const state = replicatedState(source); + const deliveries: Array<{ value: SourceValue; sequence: number }> = []; + state.subscribe((value, _context, delivery) => deliveries.push({ value, sequence: delivery.sequence })); + expect(state.value).toBe(second); + expect(deliveries).toEqual([{ value: second, sequence: 2 }]); + }); + + it("hydrates at sequence zero when attaching after existing commits", () => { + const initial = Object.freeze({ value: 0 }); + const current = Object.freeze({ value: 1 }); + const source = new TestSource(initial, 40); + source.commit(current, Object.freeze([["s", ["value"], 1] as const])); + const state = replicatedState(source); + const deliveries: number[] = []; + state.subscribe((_value, _context, delivery) => deliveries.push(delivery.sequence)); + expect(state.value).toBe(current); + expect(deliveries).toEqual([0]); + }); + + it("publishes exact source values without applying or re-diffing", () => { + const source = new TestSource(Object.freeze({ value: 0 })); + const state = replicatedState(source); + const next = Object.freeze({ value: 1 }); + // Deliberately not the operation that produces `next`: Chord must publish the source value as is. + const ops = Object.freeze([["s", ["value"], 7] as const]); + let publishedValue: SourceValue | undefined; + state.subscribe((value, _context, delivery) => { + if (delivery.kind === "update") publishedValue = value; + }); + + source.commit(next, ops); + expect(state.value).toBe(next); + expect(publishedValue).toBe(next); + }); + + it("buffers reentrant frames and skips updates covered by a late hydration", () => { + const source = new TestSource(Object.freeze({ value: 0 })); + const state = replicatedState(source); + const received: number[] = []; + const late: Array<{ kind: string; sequence: number; value: number }> = []; + let nested = false; + state.subscribe((_value, _context, delivery) => { + if (delivery.kind !== "update" || delivery.sequence !== 1 || nested) return; + nested = true; + source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); + state.subscribe((value, _context, delivery) => late.push({ ...delivery, value: value.value })); + }); + state.subscribe((value, _context, delivery) => { + if (delivery.kind === "update") received.push(value.value); + }); + + source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])); + expect(received).toEqual([1, 2]); + expect(late).toEqual([{ kind: "hydrate", sequence: 2, value: 2 }]); + }); + + it("reports listener failures without throwing them into the source", () => { + const source = new TestSource(Object.freeze({ value: 0 })); + const errors: Error[] = []; + const state = replicatedState(source, { onError: (error) => errors.push(error) }); + const received: number[] = []; + state.subscribe((_value, _context, delivery) => { + if (delivery.kind === "update") throw new Error("listener failed"); + }); + state.subscribe((value, _context, delivery) => { + if (delivery.kind === "update") received.push(value.value); + }); + + expect(() => + source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])), + ).not.toThrow(); + source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); + expect(received).toEqual([1, 2]); + expect(errors.map((error) => error.message)).toEqual(["listener failed", "listener failed"]); + }); + + it("reports cursor gaps, disposes the broken attachment, and ignores later frames", () => { + const source = new TestSource(Object.freeze({ value: 0 }), 5); + const errors: Error[] = []; + const state = replicatedState(source, { onError: (error) => errors.push(error) }); + source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const]), BACKGROUND_CONTEXT, 7); + expect(errors[0]?.message).toContain("expected 6, received 7"); + expect(source.attachments.size).toBe(0); + expect(state.value).toEqual({ value: 0 }); + source.commit(Object.freeze({ value: 3 }), Object.freeze([["s", ["value"], 3] as const]), BACKGROUND_CONTEXT, 8); + expect(state.value).toEqual({ value: 0 }); + }); + + it("keeps attachments independent and disposes each idempotently", () => { + const source = new TestSource(Object.freeze({ value: 0 })); + const first = replicatedState(source); + const second = replicatedState(source); + expect(source.attachments.size).toBe(2); + source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])); + expect(first.value.value).toBe(1); + expect(second.value.value).toBe(1); + + first.dispose(); + first.dispose(); + expect(source.attachments.size).toBe(1); + source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); + expect(first.value.value).toBe(1); + expect(second.value.value).toBe(2); + second.dispose(); + expect(source.attachments.size).toBe(0); + }); +}); diff --git a/packages/chord/test/requests.test.ts b/packages/chord/test/requests.test.ts new file mode 100644 index 000000000..4fb6c11fe --- /dev/null +++ b/packages/chord/test/requests.test.ts @@ -0,0 +1,478 @@ +// Tests for the requests in docs/requests.md (#1-#7). +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { applyImmutable } from "../src/delta/index.ts"; +import { + type Ctx, + cell, + createHost, + defineFacet, + defineRegistry, + type Facet, + type FacetApi, + type FacetLoader, + isChordError, + listener, + local, + memoryLinkPair, + method, + opaque, + remote, + resource, + type StateEvent, +} from "../src/index.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +async function until(check: () => boolean, attempts = 100): Promise { + for (let i = 0; i < attempts && !check(); i++) await sleep(2); + assert.ok(check(), "condition not reached"); +} + +function latch() { + let open = () => {}; + const promise = new Promise((resolve) => { + open = resolve; + }); + return { promise, open }; +} + +const View = Type.Object({ text: Type.String() }); + +describe("p2 requests", () => { + it("#1 a held object's state member can be returned as this object's state member", async () => { + const Conv = remote("t.conv", { + version: 1, + input: Type.Object({}), + variants: { main: { state: { view: View }, methods: { set: method(Type.String(), Type.Null()) } } }, + }); + const Chat = remote("t.chat", { + version: 1, + input: Type.Object({}), + variants: { viewer: { state: { view: View } } }, + }); + const server = createHost({ name: "server", onError: () => {} }); + server.serve(Conv, (_input, _principal, self) => { + const state = self.state({ view: cell(View, { text: "" }) }); + return { + kind: "main", + view: state.view, + set: (text, ctx) => { + state.view.replace(ctx, { text }); + return null; + }, + }; + }); + // No derivation, no cast: the handle member is the behavior member. + server.serve(Chat, (_input, _principal, self) => { + const conv = self.get(Conv, {}); + return { kind: "viewer", view: conv.view }; + }); + const chat = server.get(Chat, {}); + const values: unknown[] = []; + chat.view.subscribe((value) => values.push(value)); + await chat.ready; + await server.get(Conv, {}).set("hi"); + await tick(); + assert.deepEqual(chat.view.value, { text: "hi" }); + assert.deepEqual(values.at(-1), { text: "hi" }); + + // Across a link: the snapshot and its ops reach a browser. + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + const remoteChat = browser.get(Chat, {}, { at: "server" }); + remoteChat.view.subscribe(() => {}); + await until(() => remoteChat.view.value !== undefined); + await server.get(Conv, {}).set("there"); + await until(() => (remoteChat.view.value as { text: string } | undefined)?.text === "there"); + }); + + it("#2 facets reach their own host through FacetApi.root and FacetApi.registry", async () => { + const Config = defineRegistry("t.config"); + const Settings = local("t.settings", { version: 1, input: Type.Object({}) }); + const Reader = local("t.reader", { + version: 1, + input: Type.Object({}), + members: { read: method(Type.Null(), Type.String()) }, + }); + const captured: FacetApi[] = []; + const facet = (value: string): Facet => ({ + name: "config", + ...defineFacet((api) => { + captured.push(api); + api.serve( + Settings, + (_input, _principal, self) => { + self.register(Config, { greeting: value }); + return {}; + }, + { eager: true }, + ); + api.serve(Reader, () => ({ + read: (_args: null, ctx: Ctx) => api.registry(Config).get("greeting", ctx) ?? "", + })); + }), + }); + const host = createHost({ name: "worker", onError: () => {} }); + host.install(facet("v1")); + await sleep(5); + const api = captured[0]!; + assert.equal(api.name, "worker"); + // A callback outside Chord (a Durable tool, say) starts its own root. + assert.equal(await api.root((ctx) => ctx.get(Reader, {}).read(null, ctx)), "v1"); + await host.reload([facet("v2")]); + await sleep(5); + assert.equal(captured[1]!.registry(Config).get("greeting"), "v2"); + assert.equal(await api.root((ctx) => ctx.get(Reader, {}).read(null, ctx)), "v2"); + }); + + it("#3 a listener keeps its subscription across reloads when resource and select are unchanged", async () => { + const Watcher = local("t.watcher", { + version: 1, + input: Type.Object({}), + members: { seen: Type.Array(Type.String()) }, + }); + // A source that accepts one subscriber at a time (like a Durable watch). + let subscribers = 0; + let emit = (_event: string) => {}; + const single = { + start: (fn: (event: string) => void) => { + if (subscribers > 0) throw new Error("Watch is already started"); + subscribers++; + emit = fn; + return () => { + subscribers--; + }; + }, + }; + const facet = (tag: string): Facet => ({ + name: "watcher", + setup: (api) => + api.serve(Watcher, (_input, _principal, self) => { + const state = self.state({ seen: cell(Type.Array(Type.String()), []) }); + const res = self.resources({ watch: resource({ rev: 1, open: () => single }) }); + self.listen({ + // A new adapter object per call, and a new arrow per factory run. + events: listener( + res.watch, + (watch) => ({ subscribe: (fn: (event: string) => void) => watch.start(fn) }), + (event, ctx) => state.seen.change(ctx, (draft) => void draft.push(`${tag}:${event}`)), + ), + }); + return { seen: state.seen }; + }), + }); + const host = createHost({ onError: () => {} }); + host.install(facet("v1")); + const handle = host.get(Watcher, {}); + await handle.ready; + emit("a"); + await host.reload([facet("v2")]); + await host.reload([facet("v3")]); + emit("b"); + await sleep(5); + assert.equal(subscribers, 1); + assert.deepEqual(handle.seen.value, ["v1:a", "v3:b"]); + }); + + it("#3 a changed select resubscribes", async () => { + const Watcher = local("t.watcher2", { version: 1, input: Type.Object({}) }); + let subscribes = 0; + const source = { + subscribe: () => { + subscribes++; + return () => {}; + }, + }; + const facet = (selectText: boolean): Facet => ({ + name: "watcher2", + setup: (api) => + api.serve(Watcher, (_input, _principal, self) => { + const res = self.resources({ s: resource({ rev: 1, open: () => source }) }); + self.listen({ s: listener(res.s, selectText ? (s) => s : (value) => value, () => {}) }); + return {}; + }), + }); + const host = createHost({ onError: () => {} }); + host.install(facet(true)); + await host.get(Watcher, {}).ready; + await host.reload([facet(true)]); + assert.equal(subscribes, 1); + await host.reload([facet(false)]); + assert.equal(subscribes, 2); + }); + + it("#4 replace diffs a new revision that shares structure; unrelated values replace", async () => { + const Mirror = local("t.mirror", { + version: 1, + input: Type.Object({}), + members: { doc: Type.Unknown(), set: method(Type.Unknown(), Type.Null()) }, + }); + const Doc = Type.Object({ entries: Type.Array(Type.String()), status: Type.String() }); + let lastOptions: { diff?: boolean } | undefined; + const host = createHost({ onError: () => {} }); + host.serve(Mirror, (_input, _principal, self) => { + const state = self.state({ doc: cell(Doc, { entries: ["a", "b"], status: "idle" }) }); + return { + doc: state.doc, + set: (value: unknown, ctx: Ctx) => { + state.doc.replace(ctx, value as never, lastOptions); + return null; + }, + }; + }); + const handle = host.get(Mirror, {}); + await handle.ready; + const events: StateEvent[] = []; + let replica: unknown; + handle.doc.events((event) => { + replica = event.type === "ops" ? applyImmutable(replica, event.ops) : event.value; + events.push(event); + }); + const opsOf = (event: StateEvent | undefined) => (event?.type === "ops" ? event.ops : []); + const current = handle.doc.value as { entries: string[]; status: string }; + // A new revision reusing `entries`: a diff. + await handle.set({ entries: current.entries, status: "running" }); + assert.deepEqual(opsOf(events.at(-1)), [["s", ["status"], "running"]]); + // Unrelated containers: one replacement op. + await handle.set({ entries: ["a", "b", "c"], status: "running" }); + assert.equal(opsOf(events.at(-1))[0]?.[0], "r"); + // Forced either way. + lastOptions = { diff: true }; + await handle.set({ entries: ["a", "b", "c", "d"], status: "running" }); + assert.notEqual(opsOf(events.at(-1))[0]?.[0], "r"); + lastOptions = { diff: false }; + const shared = handle.doc.value as { entries: string[] }; + await handle.set({ entries: shared.entries, status: "done" }); + assert.equal(opsOf(events.at(-1))[0]?.[0], "r"); + await tick(); + assert.deepEqual(replica, handle.doc.value); + }); + + it("#5 roots of a child that connected behind the server are admitted in the server's epoch", async () => { + const Echo = remote("t.echo", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { epoch: method(Type.Null(), Type.Number()) } } }, + }); + const server = createHost({ name: "server", onError: () => {} }); + const echoFacet: Facet = { + name: "echo", + setup: (api) => api.serve(Echo, () => ({ kind: "main", epoch: (_args: null, ctx: Ctx) => ctx.epoch })), + }; + server.install(echoFacet); + await server.reload([echoFacet]); + assert.equal(server.epoch, 2); + assert.equal(server.previousEpoch, undefined); + const loader: FacetLoader = async () => []; + const worker = createHost({ name: "session:S1", loader, onError: () => {} }); + const go = latch(); + // A worker root of epoch 1 (an eager object's listener, say) calls the server after connecting. + const call = worker.root(async (ctx) => { + await go.promise; + return ctx.get(Echo, {}, { at: "server" }).epoch(null, ctx); + }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "S1" }, kind: "host", name: "session:S1" }); + worker.connect(up); + go.open(); + assert.equal(await call, 2); + await until(() => worker.epoch === 2); + // Unknown epochs from in-step peers stay superseded. + await assert.rejects( + server.root(() => 0, { epoch: 1 }), + (error) => isChordError(error, "superseded"), + ); + }); + + it("#6 self.expose: an opaque member backed by a resource, readable once live", async () => { + const Store = local("t.store", { + version: 1, + input: Type.Object({}), + members: { db: opaque<{ readonly queries: string[] }>("t.db@1") }, + }); + const gate = latch(); + const native = { queries: [] as string[] }; + const host = createHost({ onError: () => {} }); + host.serve(Store, (_input, _principal, self) => { + const res = self.resources({ + db: resource({ + rev: 1, + open: async () => { + await gate.promise; + return native; + }, + }), + }); + return { db: self.expose(res.db) }; + }); + const handle = host.get(Store, {}); + assert.equal(handle.db, undefined); + gate.open(); + await handle.ready; + assert.equal(handle.db, native); + + const Other = local("t.other", { version: 1, input: Type.Object({}) }); + host.serve(Other, (_input, _principal, self) => { + self.expose({ key: "x", value: 1 }); + return {}; + }); + await assert.rejects(host.get(Other, {}).ready, /expose takes a resource declared by this record/); + }); + + it("#7 a cluster reload reports what each child prepared", async () => { + const Greeter = remote("t.greeter", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + }); + const greeter = (text: string): Facet => ({ + name: "greeter", + setup: (api) => api.serve(Greeter, () => ({ kind: "main", greet: (name: string) => `${text} ${name}` })), + }); + const modules = new Map([["greeter", greeter("v1")]]); + const loader: FacetLoader = async (names) => names.map((name) => modules.get(String(name))!); + const server = createHost({ name: "server", onError: () => {} }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "S1" }, kind: "host", name: "session:S1" }); + const worker = createHost({ name: "session:S1", upstream: up, loader, onError: () => {} }); + worker.install(greeter("v1")); + await tick(); + const handle = server.get(Greeter, {}, { at: "session:S1" }); + assert.equal(await handle.greet("a"), "v1 a"); + modules.set("greeter", greeter("v2")); + const report = await server.reload([], { remote: { "session:": ["greeter"] } }); + assert.deepEqual(report.changed, []); + const child = report.children?.["session:S1"]; + assert.deepEqual(child?.changed, ["t.greeter"]); + assert.equal(child?.prepared.length, 1); + assert.equal(child?.epoch, report.epoch); + }); + + it("#8 a remote state member forgets its value when nobody listens; the next subscriber sees the fresh value", async () => { + const Counter = remote("t.count8", { + version: 1, + input: Type.Object({}), + variants: { main: { state: { n: Type.Number() }, methods: { set: method(Type.Number(), Type.Null()) } } }, + }); + const server = createHost({ name: "server", onError: () => {} }); + server.serve(Counter, (_input, _principal, self) => { + const state = self.state({ n: cell(Type.Number(), 0) }); + return { + kind: "main", + n: state.n, + set: (value, ctx) => { + state.n.replace(ctx, value); + return null; + }, + }; + }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + const counter = browser.get(Counter, {}, { at: "server" }); + const stop = counter.n.subscribe(() => {}); + await counter.set(1); + await until(() => counter.n.value === 1); + stop(); + assert.equal(counter.n.value, undefined); + await counter.set(2); + const seen: unknown[] = []; + const again = counter.n.subscribe((value) => seen.push(value)); + await until(() => seen.at(-1) === 2); + assert.deepEqual(seen, [undefined, 2]); + again(); + }); + + it("#9 handles report status changes: live, loading while a link reconnects, gone, released", async () => { + const Thing = remote("t.thing9", { version: 1, input: Type.Object({}), variants: { main: {} } }); + const server = createHost({ name: "server", onError: () => {} }); + server.serve(Thing, () => ({ kind: "main" })); + const localThing = server.get(Thing, {}, { principal: { user: "u" } }); + const localStatus: unknown[] = []; + localThing.onStatus((status, error) => + localStatus.push(status === "gone" ? [status, (error as { code: string }).code] : status), + ); + await localThing.ready; + await tick(); + await server.invalidatePrincipal({ user: "u" }); + await tick(); + assert.deepEqual(localStatus, ["live", ["gone", "gone"]]); + + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { guest: "g" } }); + const browser = createHost({ name: "browser", kind: "client", upstream: up, onError: () => {} }); + const remoteThing = browser.get(Thing, {}, { at: "server" }); + const remoteStatus: string[] = []; + remoteThing.onStatus((status) => remoteStatus.push(status)); + await remoteThing.ready; + up.close(); + await until(() => remoteStatus.length === 2); + const [down2, up2] = memoryLinkPair(); + server.accept(down2, { principal: { guest: "g" } }); + browser.connect(up2); + await until(() => remoteStatus.length === 3); + remoteThing.release(); + await tick(); + assert.deepEqual(remoteStatus, ["live", "loading", "live", "released"]); + }); + + it("#10 host close disposes holders before what they hold, and before closing links", async () => { + const log: string[] = []; + const Remote = remote("t.remote10", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { note: method(Type.String(), Type.Null()) } } }, + }); + const server = createHost({ name: "server", onError: () => {} }); + server.serve(Remote, () => ({ + kind: "main", + note: (text) => { + log.push(`server: ${text}`); + return null; + }, + })); + const Proc = local("t.proc", { + version: 1, + input: Type.Object({}), + members: { tracker: opaque>("t.tracker@1") }, + }); + const Chat = local("t.chat10", { version: 1, input: Type.Object({}) }); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { worker: "w" }, kind: "host", name: "w" }); + const worker = createHost({ name: "w", upstream: up, onError: (error) => log.push(`error: ${String(error)}`) }); + // Eager: activated (and registered with the host) before the object that holds it. + worker.serve( + Proc, + (_input, _principal, self) => { + const res = self.resources({ + tracker: resource({ rev: 1, open: () => new Set(), close: () => void log.push("proc closed") }), + }); + return { tracker: self.expose(res.tracker) }; + }, + { eager: true }, + ); + await sleep(5); + worker.serve(Chat, (_input, _principal, self) => { + const proc = self.get(Proc, {}); + const upstream = self.get(Remote, {}, { at: "server" }); + self.resources({ + session: resource({ + rev: 1, + open: () => proc.tracker?.add("chat"), + close: async () => { + proc.tracker?.delete("chat"); + log.push(`chat closed, proc ${proc.status}, tracker ${proc.tracker === undefined ? "gone" : "ok"}`); + await upstream.note("bye"); + }, + }), + }); + return {}; + }); + await worker.get(Chat, {}).ready; + await worker.close(); + assert.deepEqual(log, ["chat closed, proc live, tracker ok", "server: bye", "proc closed"]); + }); +}); diff --git a/packages/chord/test/review.test.ts b/packages/chord/test/review.test.ts new file mode 100644 index 000000000..b615ceb8e --- /dev/null +++ b/packages/chord/test/review.test.ts @@ -0,0 +1,653 @@ +// Regression tests for Sol's phase 1a review (findings #1-#11) and the requested race tests. +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { Value } from "typebox/value"; +import { describe, it } from "vitest"; +import { + type Ctx, + cell, + createHost, + defineRegistry, + type Facet, + type Handle, + isChordError, + listener, + local, + method, + remote, + resource, +} from "../src/index.ts"; +import { fits } from "../src/schema.ts"; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +const ticks = async (n = 10) => { + for (let i = 0; i < n; i++) await tick(); +}; + +function latch() { + let open = () => {}; + const promise = new Promise((resolve) => { + open = resolve; + }); + return { promise, open }; +} + +function emitter(options: { failSubscribe?: boolean } = {}) { + const listeners = new Set<(event: E) => void>(); + return { + emit: (event: E) => { + for (const fn of [...listeners]) fn(event); + }, + subscribe: (fn: (event: E) => void) => { + if (options.failSubscribe) throw new Error("subscribe fails"); + listeners.add(fn); + return () => listeners.delete(fn); + }, + get size() { + return listeners.size; + }, + }; +} + +const Item = local("t.item", { + version: 1, + input: Type.Object({ id: Type.String() }), + members: { version: method(Type.Null(), Type.String()) }, +}); + +describe("review: reload commit", () => { + it("#1 a failing listener subscription fails prepare; the epoch and all records stay unchanged", async () => { + const host = createHost({ onError: () => {} }); + const sources = { a: emitter(), b: emitter({ failSubscribe: true }), c: emitter() }; + const facet = (version: string, listen: boolean): Facet => ({ + name: "item", + setup: (api) => + api.serve(Item, ({ id }, _principal, self) => { + if (listen) self.listen({ events: listener(sources[id as "a"], () => {}) }); + return { version: () => version }; + }), + }); + host.install(facet("v1", false)); + const handles = ["a", "b", "c"].map((id) => host.get(Item, { id })); + await Promise.all(handles.map((handle) => handle.ready)); + const epoch = host.epoch; + await assert.rejects(host.reload([facet("v2", true)]), /subscribe fails/); + assert.equal(host.epoch, epoch); + assert.deepEqual(await Promise.all(handles.map((handle) => handle.version(null))), ["v1", "v1", "v1"]); + assert.equal(sources.a.size, 0); + assert.equal(sources.c.size, 0); + }); + + it("#2 migration is validated against the new remote schemas, not the obsolete ones (dev mode)", async () => { + const host = createHost({ dev: true, onError: () => {} }); + const V1 = remote("t.counted", { + version: 1, + input: Type.Object({}), + variants: { main: { state: { count: Type.Object({ n: Type.Number() }) } } }, + }); + const V2 = remote("t.counted", { + version: 2, + input: Type.Object({}), + variants: { main: { state: { count: Type.Object({ total: Type.Number() }) } } }, + }); + host.install({ + name: "counted", + setup: (api) => + api.serve( + V1, + (_input, _principal, self) => ({ + kind: "main", + count: self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 3 }) }).count, + }), + { linger: 60_000 }, + ), + }); + const first = host.get(V1, {}); + await first.ready; + first.release(); + await host.reload([ + { + name: "counted", + setup: (api) => + api.serve( + V2, + (_input, _principal, self) => ({ + kind: "main", + count: self.state({ count: cell(Type.Object({ total: Type.Number() }), { total: 0 }) }).count, + }), + { + state: { version: 2, migrate: { count: (old) => ({ total: (old as { n: number }).n }) } }, + linger: 60_000, + }, + ), + }, + ]); + const second = host.get(V2, {}); + const values: unknown[] = []; + second.count.subscribe((value) => values.push(value)); + await second.ready; + await ticks(); + assert.deepEqual(second.count.value, { total: 3 }); + }); + + it("#1 a commit that still fails midway fails the host terminally (no half-published epoch serving work)", async () => { + const host = createHost({ onError: () => {} }); + host.install({ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v1" })) }); + const handle = host.get(Item, { id: "a" }); + await handle.ready; + // Everything fallible runs in prepare now, so break an install step directly (white box). + const records = (host as unknown as { records: Map }).records; + for (const record of records.values()) { + record.startTasks = () => { + throw new Error("install step fails"); + }; + } + await assert.rejects( + host.reload([{ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v2" })) }]), + /install step fails/, + ); + await assert.rejects( + handle.version(null), + (error) => isChordError(error, "unavailable") && error.reason === "host_failed", + ); + }); + + it("reentrant observers calling another record during a commit see the whole new epoch", async () => { + const host = createHost({ onError: () => {} }); + const Other = local("t.other", { + version: 1, + input: Type.Object({}), + members: { version: method(Type.Null(), Type.String()) }, + }); + const facet = (version: string): Facet => ({ + name: "pair", + setup: (api) => { + api.serve(Item, () => ({ version: () => version })); + api.serve(Other, () => ({ version: () => version })); + }, + }); + host.install(facet("v1")); + const item = host.get(Item, { id: "a" }); + const other = host.get(Other, {}); + await Promise.all([item.ready, other.ready]); + const seen: Promise[] = []; + item.onShape(() => seen.push(other.version(null))); + await host.reload([facet("v2")]); + assert.deepEqual(await Promise.all(seen), ["v2"]); + }); +}); + +describe("review: leases, invocations, activation", () => { + const Store = local("t.store", { + version: 1, + input: Type.Object({}), + members: { count: Type.Unknown() }, + }); + + it("#4 a running listener holds a lease: the gate waits for it before migrating", async () => { + const host = createHost({ onError: () => {} }); + const events = emitter(); + const go = latch(); + const seen: unknown[] = []; + host.install({ + name: "store", + setup: (api) => + api.serve(Store, (_input, _principal, self) => { + const state = self.state({ count: cell(Type.Object({ n: Type.Number() }), { n: 1 }) }); + self.listen({ + slow: listener(events, async () => { + await go.promise; + seen.push(state.count.value); + }), + }); + return { count: state.count }; + }), + }); + const handle = host.get(Store, {}); + await handle.ready; + events.emit(1); + await tick(); + const epoch = host.epoch; + const reload = host.reload( + [ + { + name: "store", + setup: (api) => + api.serve( + Store, + (_input, _principal, self) => ({ + count: self.state({ count: cell(Type.Object({ total: Type.Number() }), { total: 0 }) }).count, + }), + { state: { version: 2, migrate: { count: (old) => ({ total: (old as { n: number }).n }) } } }, + ), + }, + ], + { gateDeadline: 2000 }, + ); + await ticks(20); + assert.equal(host.epoch, epoch, "the gate must wait for the listener"); + go.open(); + await reload; + assert.deepEqual(seen, [{ n: 1 }]); + assert.deepEqual(handle.count.value, { total: 1 }); + }); + + it("#5 disposal waits for running methods before closing resources (released and forced)", async () => { + const Worker = local("t.busy", { + version: 1, + input: Type.Object({ id: Type.String() }), + members: { work: method(Type.Null(), Type.Boolean()) }, + }); + const log: string[] = []; + let go = latch(); + const host = createHost({ onError: () => {} }); + host.serve(Worker, ({ id }, _principal, self) => { + const res = self.resources({ + conn: resource({ + rev: 1, + open: () => ({ closed: false }), + close: (conn) => { + conn.closed = true; + log.push(`close ${id}`); + }, + }), + }); + return { + work: async () => { + await go.promise; + log.push(`done ${id}`); + return res.conn.value.closed; + }, + }; + }); + const released = host.get(Worker, { id: "r" }); + const call = released.work(null); + await ticks(); + released.release(); + await ticks(); + go.open(); + assert.equal(await call, false); + await ticks(); + assert.deepEqual(log, ["done r", "close r"]); + + go = latch(); + log.length = 0; + const forced = host.get(Worker, { id: "f" }, { principal: { user: "u" } }); + const forcedCall = forced.work(null); + await ticks(); + const invalidated = host.invalidatePrincipal((principal) => (principal as { user?: string }).user === "u"); + await ticks(); + assert.equal(forced.status, "gone"); + assert.deepEqual(log, []); + go.open(); + await invalidated; + assert.equal(await forcedCall, false); + assert.deepEqual(log, ["done f", "close f"]); + }); + + it("#6 a registry conflict found while finalizing activation unwinds the loser completely", async () => { + const errors: unknown[] = []; + const host = createHost({ onError: (error) => errors.push(error) }); + const Routes = defineRegistry("t.review-routes"); + const log: string[] = []; + const types = ["t.eager-a", "t.eager-b"].map((name) => local(name, { version: 1, input: Type.Object({}) })); + for (const type of types) { + host.serve( + type, + (_input, _principal, self) => { + self.resources({ + r: resource({ + rev: 1, + open: () => void log.push(`open ${type.name}`), + close: () => void log.push(`close ${type.name}`), + }), + }); + self.register(Routes, { same: type.name }); + return {}; + }, + { eager: true }, + ); + } + await ticks(20); + assert.equal(log.filter((entry) => entry.startsWith("open")).length, 2); + assert.equal(log.filter((entry) => entry.startsWith("close")).length, 1); + assert.equal(host.registry(Routes).table().size, 1); + assert.ok(errors.some((error) => isChordError(error, "invalid") && error.reason === "registry_conflict")); + }); + + it("#6 releasing the last hold while listeners start up does not resurrect the record", async () => { + const host = createHost({ onError: () => {} }); + const log: string[] = []; + let handle: Handle | undefined; + let runs = 0; + host.serve(Store, (_input, _principal, self) => { + runs++; + const state = self.state({ count: cell(Type.Number(), 0) }); + self.resources({ + r: resource({ rev: 1, open: () => void log.push("open"), close: () => void log.push("close") }), + }); + // A readable delivers its current value as the listener starts: release the only hold right then. + self.listen({ drop: listener(state.count, () => handle?.release()) }); + return { count: state.count }; + }); + handle = host.get(Store, {}); + await handle.ready.catch(() => {}); + await ticks(20); + assert.deepEqual(log, ["open", "close"]); + const again = host.get(Store, {}); + handle = undefined; + await again.ready; + assert.equal(runs, 2); + }); +}); + +describe("review: prepare isolation and contracts", () => { + const Db = local("t.db", { + version: 1, + input: Type.Object({}), + members: { read: method(Type.Null(), Type.Unknown()) }, + }); + + it("#7 a prepare rerun cannot touch retained resources, even open ones", async () => { + const host = createHost({ onError: () => {} }); + const native = { mutated: false }; + const facet = (touch: boolean): Facet => ({ + name: "db", + setup: (api) => + api.serve(Db, (_input, _principal, self) => { + const res = self.resources({ db: resource({ rev: 1, open: () => native }) }); + if (touch) res.db.value.mutated = true; + return { read: () => res.db.value }; + }), + }); + host.install(facet(false)); + const handle = host.get(Db, {}); + await handle.ready; + await assert.rejects(host.reload([facet(true)]), (error) => isChordError(error, "not_ready")); + assert.equal(native.mutated, false); + assert.deepEqual(await handle.read(null), { mutated: false }); + }); + + it("#7 records of types a reload adds start listeners and tasks only at commit", async () => { + const host = createHost({ onError: () => {} }); + const log: string[] = []; + const Fresh = local("t.fresh", { version: 1, input: Type.Object({}) }); + host.install({ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v1" })) }); + await Promise.all(["a", "b"].map((id) => host.get(Item, { id }).ready)); + const plugins = (fail: boolean): Facet[] => [ + { + name: "fresh", + setup: (api) => + api.serve(Fresh, (_input, _principal, self) => { + self.tasks({ + run: async () => { + log.push("task started"); + }, + }); + return {}; + }), + }, + { + name: "item", + setup: (api) => + api.serve(Item, ({ id }, _principal, self) => { + self.get(Fresh, {}); + if (fail && id === "b") throw new Error("b fails"); + return { version: () => "v2" }; + }), + }, + ]; + await assert.rejects(host.reload(plugins(true)), /b fails/); + await ticks(); + assert.deepEqual(log, []); + await host.reload(plugins(false)); + await ticks(); + assert.deepEqual(log, ["task started"]); + }); + + it("#8 a breaking provider change and its updated consumer reload in one batch", async () => { + const host = createHost({ onError: () => {} }); + const P1 = remote("t.provider", { + version: 1, + input: Type.Object({}), + variants: { main: { methods: { get: method(Type.Null(), Type.String()) } } }, + }); + const P2 = remote("t.provider", { + version: 2, + input: Type.Object({}), + variants: { main: { methods: { fetch: method(Type.Null(), Type.String()) } } }, + }); + const Consumer = local("t.consumer", { + version: 1, + input: Type.Object({}), + members: { ask: method(Type.Null(), Type.String()) }, + }); + host.install({ + name: "both", + setup: (api) => { + api.serve(P1, () => ({ kind: "main", get: () => "old" })); + api.serve(Consumer, (_input, _principal, self) => { + const provider = self.get(P1, {}); + return { ask: (_args: null, ctx: Ctx) => provider.get(null, ctx) }; + }); + }, + }); + const consumer = host.get(Consumer, {}); + assert.equal(await consumer.ask(null), "old"); + await host.reload([ + { + name: "both", + setup: (api) => { + api.serve(P2, () => ({ kind: "main", fetch: () => "new" })); + api.serve(Consumer, (_input, _principal, self) => { + const provider = self.get(P2, {}); + return { ask: (_args: null, ctx: Ctx) => provider.fetch(null, ctx) }; + }); + }, + }, + ]); + assert.equal(await consumer.ask(null), "new"); + }); + + it("#9 an exclusive resource whose close failed is restored on rollback, or marked unavailable", async () => { + for (const reopen of ["works", "fails"] as const) { + const host = createHost({ onError: () => {} }); + const log: string[] = []; + let opens = 0; + const facet = (rev: number): Facet => ({ + name: "db", + setup: (api) => + api.serve(Db, (_input, _principal, self) => { + const res = self.resources({ + port: resource({ + rev, + exclusive: true, + open: () => { + if (rev === 2) throw new Error("new port fails"); + if (++opens > 1 && reopen === "fails") throw new Error("reopen fails"); + log.push(`open ${rev}`); + return { rev }; + }, + close: () => { + log.push(`close ${rev}`); + throw new Error("close fails"); + }, + }), + }); + return { read: () => res.port.value }; + }), + }); + host.install(facet(1)); + const handle = host.get(Db, {}); + await handle.ready; + // The failing close aborts the reload at the gate; the old resource must not stay closed. + await assert.rejects(host.reload([facet(2)]), /close fails/); + if (reopen === "works") { + assert.deepEqual(log, ["open 1", "close 1", "open 1"]); + assert.deepEqual(await handle.read(null), { rev: 1 }); + } else { + await assert.rejects(handle.read(null), (error) => isChordError(error, "unavailable")); + } + } + }); + + it("#10 retirement closes dropped resources before releasing dropped holds", async () => { + const host = createHost({ onError: () => {} }); + const log: string[] = []; + const Dep = local("t.dep", { version: 1, input: Type.Object({}) }); + const facet = (withDecls: boolean): Facet => ({ + name: "item", + setup: (api) => { + api.serve(Dep, (_input, _principal, self) => { + self.resources({ r: resource({ rev: 1, open: () => {}, close: () => void log.push("dep closed") }) }); + return {}; + }); + api.serve(Item, (_input, _principal, self) => { + if (withDecls) { + self.get(Dep, {}); + // Slow close: releasing the hold first would let the dependency close before it finishes. + self.resources({ + own: resource({ + rev: 1, + open: () => {}, + close: async () => { + await ticks(3); + log.push("own closed"); + }, + }), + }); + } + return { version: () => (withDecls ? "v1" : "v2") }; + }); + }, + }); + host.install(facet(true)); + const handle = host.get(Item, { id: "a" }); + await handle.ready; + const go = latch(); + const pinned = host.root(async (ctx) => { + await handle.version(null, ctx); + await go.promise; + }); + await tick(); + await host.reload([facet(false)]); + assert.deepEqual(log, []); + go.open(); + await pinned; + await ticks(20); + assert.deepEqual(log, ["own closed", "dep closed"]); + }); + + it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with Value.Check", () => { + const cases = [ + [Type.String(), Type.String({ not: { const: "x" } }), "x"], + [Type.Array(Type.Number()), Type.Array(Type.Number(), { contains: Type.Literal(1) }), [2]], + [ + Type.Object({ a: Type.String() }), + Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), + { a: "s", b: "t" }, + ], + [ + Type.Object({ x: Type.String() }), + Type.Object({}, { patternProperties: { "^x$": Type.Number() } }), + { x: "s" }, + ], + [Type.Number(), Type.Intersect([Type.Number(), Type.Number({ minimum: 1 })]), 0], + [ + Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), + Type.Object({ a: Type.String(), b: Type.Optional(Type.String()) }), + { a: "", b: 1 }, + ], + ] as const; + for (const [sub, sup, value] of cases) { + assert.ok(Value.Check(sub, value) && !Value.Check(sup, value), `bad counterexample ${JSON.stringify(value)}`); + assert.equal(fits(sub, sup), false, `fits(${JSON.stringify(sub)}, ${JSON.stringify(sup)})`); + } + // Additive changes still fit. + assert.ok( + fits(Type.Object({ a: Type.String() }), Type.Object({ a: Type.String(), b: Type.Optional(Type.Number()) })), + ); + assert.ok(fits(Type.Object({ a: Type.String(), extra: Type.Number() }), Type.Object({ a: Type.String() }))); + }); + + it("#11 arguments are stripped to what the consumer knows, then to what the provider knows", async () => { + const host = createHost({ onError: () => {} }); + const Strict = remote("t.strict", { + version: 1, + input: Type.Object({}), + variants: { + main: { + methods: { + echo: method(Type.Object({ a: Type.String() }, { additionalProperties: false }), Type.Unknown()), + }, + }, + }, + }); + host.serve(Strict, () => ({ kind: "main", echo: (args) => args })); + const handle = host.get(Strict, {}); + assert.deepEqual(await handle.echo({ a: "x", junk: 1 } as never), { a: "x" }); + }); +}); + +describe("review: ctx races", () => { + const Target = local("t.target", { + version: 1, + input: Type.Object({}), + members: { hit: method(Type.Null(), Type.Null()) }, + }); + + it("an aborted task ctx cannot start fresh calls", async () => { + const host = createHost({ onError: () => {} }); + const Runner = local("t.runner", { version: 1, input: Type.Object({}) }); + let hits = 0; + const go = latch(); + let outcome: unknown; + host.serve(Target, () => ({ + hit: () => { + hits++; + return null; + }, + })); + host.serve(Runner, (_input, _principal, self) => { + const target = self.get(Target, {}); + self.tasks({ + run: async (ctx) => { + await go.promise; + outcome = await target.hit(null, ctx).catch((error: unknown) => error); + }, + }); + return {}; + }); + const runner = host.get(Runner, {}); + await runner.ready; + const target = host.get(Target, {}); + await target.ready; + runner.release(); + await ticks(); + go.open(); + await ticks(20); + assert.equal(hits, 0); + assert.ok(isChordError(outcome, "aborted")); + }); + + it("an expired ctx and a foreign ctx are refused", async () => { + const host = createHost({ onError: () => {} }); + const other = createHost({ onError: () => {} }); + host.serve(Target, () => ({ hit: () => null })); + const target = host.get(Target, {}); + await target.ready; + const expired = await host.root((ctx) => ctx); + await assert.rejects( + target.hit(null, expired), + (error) => isChordError(error, "aborted") && error.reason === "expired", + ); + const foreign = await other.root(async (ctx) => { + await assert.rejects( + target.hit(null, ctx), + (error) => isChordError(error, "invalid") && error.reason === "foreign_ctx", + ); + return true; + }); + assert.ok(foreign); + }); +}); diff --git a/packages/chord/test/service-delivery.test.ts b/packages/chord/test/service-delivery.test.ts deleted file mode 100644 index 3fd03a146..000000000 --- a/packages/chord/test/service-delivery.test.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { BACKGROUND_CONTEXT, withCancel } from "../src/context/index.ts"; -import { - type Context, - createRemoteServiceBinding, - createServiceStateDecoder, - createServiceStateEncoder, - defineService, - parseWireServiceProviderUpdate, - parseWireServiceSubscriptionSnapshot, - RemoteServiceProvider, - type RemoteServiceTransport, - type ReplicatedState, - replicatedState, - type ServiceProviderUpdate, -} from "../src/index.ts"; -import { getReplicatedStateInternals } from "../src/services/state-internals.ts"; - -interface Counter { - readonly state: ReplicatedState<{ value: number }>; -} -const Counter = defineService("test.delivery-counter"); -interface Pair { - readonly left: ReplicatedState<{ value: number }>; - readonly right: ReplicatedState<{ value: number }>; -} -const Pair = defineService("test.delivery-pair"); - -function wireTransport( - provider: RemoteServiceProvider, - beforeActivate: () => void, - updates: ServiceProviderUpdate[], -): RemoteServiceTransport { - return { - invoke: (call, context) => provider.invoke(call, context), - async subscribe(serviceId, mode, listener) { - const encoder = createServiceStateEncoder(); - const decoder = createServiceStateDecoder(); - const subscription = provider.subscribe(serviceId, mode, (update, context) => { - const decoded = decoder.decodeUpdate(parseWireServiceProviderUpdate(encoder.encodeUpdate(update))); - updates.push(decoded); - listener(decoded, context); - }); - const snapshot = decoder.decodeSnapshot( - parseWireServiceSubscriptionSnapshot(encoder.encodeSnapshot(subscription.snapshot)), - ); - return { - snapshot, - activate() { - beforeActivate(); - subscription.activate(); - }, - close: () => subscription.close(), - }; - }, - }; -} - -describe("provider delivery queues", () => { - test("appends reentrant updates to the activation FIFO", () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const received: number[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update) => { - if (update.type !== "state") return; - received.push(update.sequence); - if (update.sequence === 1) state.replace(BACKGROUND_CONTEXT, { value: 3 }); - }); - state.replace(BACKGROUND_CONTEXT, { value: 1 }); - state.replace(BACKGROUND_CONTEXT, { value: 2 }); - subscription.activate(); - subscription.activate(); - expect(received).toEqual([1, 2, 3]); - provider.dispose(); - }); - - test.each([100, 101, 102, 201, 202])("bounds %i pending updates with explicit root resets", (count) => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const updates: ServiceProviderUpdate[] = []; - const contexts: Context[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update, context) => { - updates.push(update); - contexts.push(context); - }); - const { context } = withCancel(BACKGROUND_CONTEXT); - for (let value = 1; value <= count; value += 1) state.replace(context, { value }); - expect(updates).toEqual([]); - expect(subscription.snapshot.instances[0]?.members[0]).toMatchObject({ sequence: 0 }); - subscription.activate(); - const first = Math.floor((count - 1) / 100) * 100 + 1; - expect(updates).toHaveLength(count - first + 1); - if (count > 100) { - expect(updates[0]).toEqual({ - type: "reset", - snapshot: { - serviceId: Counter.id, - mode: "singleton", - instances: [ - { members: [{ name: "state", kind: "state", sequence: first, ops: [["r", { value: first }]] }] }, - ], - }, - }); - } - expect(contexts.every((received) => received === context)).toBe(true); - state.replace(context, { value: count + 1 }); - expect(updates.at(-1)).toMatchObject({ type: "state", sequence: count + 1 }); - provider.dispose(); - }); - - test("does not compact the running delivery when activation overflows reentrantly", () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const updates: ServiceProviderUpdate[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update) => { - updates.push(update); - if (update.type === "state" && update.sequence === 1) { - for (let value = 3; value <= 103; value += 1) state.replace(BACKGROUND_CONTEXT, { value }); - } - }); - state.replace(BACKGROUND_CONTEXT, { value: 1 }); - state.replace(BACKGROUND_CONTEXT, { value: 2 }); - subscription.activate(); - expect(updates).toHaveLength(3); - expect(updates[0]).toMatchObject({ type: "state", sequence: 1 }); - expect(updates[1]).toMatchObject({ - type: "reset", - snapshot: { instances: [{ members: [{ sequence: 102, ops: [["r", { value: 102 }]] }] }] }, - }); - expect(updates[2]).toMatchObject({ type: "state", sequence: 103 }); - provider.dispose(); - }); - - test("suppresses notifications already covered by an overflow snapshot", () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - getReplicatedStateInternals(state)!.subscribe((_ops, sequence) => { - if (sequence === 101) { - state.replace(BACKGROUND_CONTEXT, { value: 102 }); - state.replace(BACKGROUND_CONTEXT, { value: 103 }); - } - }); - provider.provide(Counter, { state }); - const updates: ServiceProviderUpdate[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update) => updates.push(update)); - for (let value = 1; value <= 101; value += 1) state.replace(BACKGROUND_CONTEXT, { value }); - subscription.activate(); - expect(updates).toHaveLength(1); - expect(updates[0]).toMatchObject({ - type: "reset", - snapshot: { instances: [{ members: [{ sequence: 103, ops: [["r", { value: 103 }]] }] }] }, - }); - state.replace(BACKGROUND_CONTEXT, { value: 104 }); - expect(updates[1]).toMatchObject({ type: "state", sequence: 104 }); - provider.dispose(); - }); - - test("preserves lifecycle ordering across subscribers during reentrant publication", () => { - const provider = new RemoteServiceProvider([Counter]); - provider.provide(Counter, { state: replicatedState({ value: 0 }) }); - const first: ServiceProviderUpdate[] = []; - const second: ServiceProviderUpdate[] = []; - provider - .subscribe(Counter.id, "singleton", (update) => { - first.push(update); - if (first.length === 1) provider.replace(Counter, { state: replicatedState({ value: 2 }) }); - }) - .activate(); - provider.subscribe(Counter.id, "singleton", (update) => second.push(update)).activate(); - provider.replace(Counter, { state: replicatedState({ value: 1 }) }); - expect(first).toEqual(second); - expect(second).toHaveLength(2); - expect(second[0]).toMatchObject({ type: "replaced", snapshot: { members: [{ ops: [["r", { value: 1 }]] }] } }); - provider.dispose(); - }); - - test("close during a callback prevents later buffered callbacks", () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const received: ServiceProviderUpdate[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update) => { - received.push(update); - subscription.close(); - }); - state.replace(BACKGROUND_CONTEXT, { value: 1 }); - state.replace(BACKGROUND_CONTEXT, { value: 2 }); - subscription.activate(); - expect(received).toHaveLength(1); - provider.dispose(); - }); - - test("drains terminal updates when disposed reentrantly", () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const received: ServiceProviderUpdate[] = []; - const subscription = provider.subscribe(Counter.id, "singleton", (update) => { - received.push(update); - if (update.type === "state") provider.dispose(); - }); - state.replace(BACKGROUND_CONTEXT, { value: 1 }); - subscription.activate(); - expect(received.map((update) => update.type)).toEqual(["state", "unavailable"]); - }); -}); - -test("rebaselines every member through wire codecs, then resumes contiguous deltas", async () => { - const provider = new RemoteServiceProvider([Pair]); - const left = replicatedState({ value: 0 }); - const right = replicatedState({ value: 0 }); - provider.provide(Pair, { left, right }); - const errors: Error[] = []; - const updates: ServiceProviderUpdate[] = []; - const namespace = createRemoteServiceBinding({ - services: [Pair], - transport: wireTransport( - provider, - () => { - for (let value = 1; value <= 101; value += 1) { - left.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = value; - }); - right.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = value; - }); - } - }, - updates, - ), - onError: (error) => errors.push(error), - }); - const pair = namespace.use(Pair); - await namespace.ready(BACKGROUND_CONTEXT); - expect(updates.map((update) => update.type)).toEqual(["reset", "state"]); - expect(pair.left.value).toEqual({ value: 101 }); - expect(pair.right.value).toEqual({ value: 101 }); - for (let value = 102; value <= 104; value += 1) { - left.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = value; - }); - right.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = value; - }); - } - expect(pair.left.value).toEqual({ value: 104 }); - expect(pair.right.value).toEqual({ value: 104 }); - expect(errors).toEqual([]); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); -}); - -test("an overflow reset can make a singleton unavailable before a later replacement", async () => { - const provider = new RemoteServiceProvider([Counter]); - provider.provide(Counter, { state: replicatedState({ value: 0 }) }); - const updates: ServiceProviderUpdate[] = []; - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [Counter], - transport: wireTransport( - provider, - () => { - for (let value = 1; value <= 100; value += 1) - provider.replace(Counter, { state: replicatedState({ value }) }); - provider.withdraw(Counter); - }, - updates, - ), - onError: (error) => errors.push(error), - }); - const counter = namespace.use(Counter); - await namespace.ready(BACKGROUND_CONTEXT); - expect(updates).toEqual([{ type: "reset", snapshot: { serviceId: Counter.id, mode: "singleton", instances: [] } }]); - expect(counter.state.value).toBeUndefined(); - const replacement = replicatedState({ value: 200 }); - provider.replace(Counter, { state: replacement }); - replacement.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 201; - }); - expect(namespace.use(Counter)).toBe(counter); - expect(counter.state.value).toEqual({ value: 201 }); - expect(errors).toEqual([]); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); -}); - -test("keyed resets retain live generations and reconcile closed and reused keys", async () => { - const provider = new RemoteServiceProvider([{ service: Counter, mode: "keyed" }]); - const retained = replicatedState({ value: 0 }); - provider.spawn(Counter, "retained", { state: retained }); - const closeOld = provider.spawn(Counter, "reused", { state: replicatedState({ value: 0 }) }); - const removed = provider.spawn(Counter, "removed", { state: replicatedState({ value: 0 }) }); - const updates: ServiceProviderUpdate[] = []; - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [Counter], - transport: wireTransport(provider, () => {}, updates), - onError: (error) => errors.push(error), - }); - const observed: { service: Counter; context: Context }[] = []; - namespace.observe(Counter, (service, context) => { - observed.push({ service, context }); - }); - await namespace.ready(BACKGROUND_CONTEXT); - expect(observed).toHaveLength(3); - const stable = observed[1]!; // Provider snapshots are sorted by key. - const old = observed[2]!; - const replacement = replicatedState({ value: 500 }); - stable.service.state.subscribe((value) => { - if (value.value !== 1) return; - closeOld(); - removed(); - provider.spawn(Counter, "reused", { state: replacement }); - retained.replace(BACKGROUND_CONTEXT, { value: 102 }); - for (let next = 501; next <= 601; next += 1) replacement.replace(BACKGROUND_CONTEXT, { value: next }); - }); - retained.replace(BACKGROUND_CONTEXT, { value: 1 }); - expect(updates.some((update) => update.type === "reset")).toBe(true); - expect(observed).toHaveLength(4); - expect(stable.context.abortSignal?.aborted).toBe(false); - expect(stable.service.state.value).toEqual({ value: 102 }); - expect(old.context.abortSignal?.aborted).toBe(true); - expect(observed[0]?.context.abortSignal?.aborted).toBe(true); - expect(observed[3]?.service.state.value).toEqual({ value: 601 }); - replacement.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 602; - }); - expect(observed[3]?.service.state.value).toEqual({ value: 602 }); - expect(errors).toEqual([]); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); -}); diff --git a/packages/chord/test/service-wire.test.ts b/packages/chord/test/service-wire.test.ts deleted file mode 100644 index 958a017bb..000000000 --- a/packages/chord/test/service-wire.test.ts +++ /dev/null @@ -1,322 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; -import { - createRemoteServiceEndpoint, - createServiceCatalogueCall, - createServiceStateDecoder, - createServiceStateEncoder, - createServiceSubscribeCall, - createServiceUnsubscribeCall, - decodeServiceControlCall, - defineService, - parseServiceCall, - parseServiceCatalogue, - parseServiceProviderUpdate, - parseServiceSubscriptionSnapshot, - parseWireServiceProviderUpdate, - parseWireServiceSubscriptionSnapshot, - RemoteServiceProvider, - type ReplicatedState, - replicatedState, - type ServiceProviderUpdate, - type ServiceSubscriptionSnapshot, -} from "../src/index.ts"; - -describe("service wire protocol", () => { - test("encodes control calls and validates service values", () => { - expect(decodeServiceControlCall(createServiceCatalogueCall())).toEqual({ type: "catalogue" }); - expect( - parseServiceCatalogue([ - { serviceId: "pi.models", mode: "singleton" }, - { serviceId: "pi.dialogs", mode: "keyed" }, - ]), - ).toEqual([ - { serviceId: "pi.models", mode: "singleton" }, - { serviceId: "pi.dialogs", mode: "keyed" }, - ]); - const subscribe = createServiceSubscribeCall("subscription-1", "pi.models", "singleton"); - expect(decodeServiceControlCall(subscribe)).toEqual({ - type: "subscribe", - subscriptionId: "subscription-1", - serviceId: "pi.models", - mode: "singleton", - }); - expect(decodeServiceControlCall(createServiceUnsubscribeCall("subscription-1"))).toEqual({ - type: "unsubscribe", - subscriptionId: "subscription-1", - }); - expect( - parseServiceCall({ - serviceId: "pi.question-dialog", - instance: { key: "invocation-1", generation: 2 }, - member: "submit", - args: [{ outcome: "selected", index: 0 }], - }), - ).toMatchObject({ member: "submit" }); - }); - - test("rejects malformed service values", () => { - expect(() => parseServiceCall({ serviceId: "pi.models", member: "list", args: [], extra: true })).toThrow( - "Invalid service call", - ); - expect(() => parseServiceCatalogue([{ serviceId: "pi.models", mode: "unknown" }])).toThrow( - "Invalid service catalogue", - ); - expect(() => parseServiceProviderUpdate({ type: "state", member: "state", sequence: 0, ops: [] })).toThrow( - "Invalid service state update", - ); - expect(() => - parseWireServiceProviderUpdate({ type: "state", member: "state", sequence: 1, ops: [["?", 0]] }), - ).toThrow(); - }); - - test("validates decoded and wire snapshots and updates", () => { - const snapshot: ServiceSubscriptionSnapshot = { - serviceId: "pi.models", - mode: "singleton", - instances: [ - { - members: [{ name: "state", kind: "state", sequence: 0, ops: [["r", { revision: 1 }]] }], - }, - ], - }; - expect(parseServiceSubscriptionSnapshot(snapshot)).toBe(snapshot); - const encoder = createServiceStateEncoder(); - const wireSnapshot = encoder.encodeSnapshot(snapshot); - expect(parseWireServiceSubscriptionSnapshot(wireSnapshot)).toBe(wireSnapshot); - const update: ServiceProviderUpdate = { - type: "state", - member: "state", - sequence: 1, - ops: [["s", ["revision"], 2]], - }; - expect(parseServiceProviderUpdate(update)).toBe(update); - const wireUpdate = encoder.encodeUpdate(update); - expect(parseWireServiceProviderUpdate(wireUpdate)).toBe(wireUpdate); - }); - - test("keeps one operation codec pair for one subscription state", () => { - const enc = createServiceStateEncoder(); - const dec = createServiceStateDecoder(); - const snapshot: ServiceSubscriptionSnapshot = { - serviceId: "pi.models", - mode: "singleton", - instances: [ - { - members: [{ name: "state", kind: "state", sequence: 0, ops: [["r", { revision: 0 }]] }], - }, - ], - }; - expect(dec.decodeSnapshot(enc.encodeSnapshot(snapshot))).toEqual(snapshot); - - const first: ServiceProviderUpdate = { - type: "state", - member: "state", - sequence: 1, - ops: [["s", ["revision"], 1]], - }; - const second: ServiceProviderUpdate = { - type: "state", - member: "state", - sequence: 2, - ops: [["s", ["revision"], 2]], - }; - const firstWire = enc.encodeUpdate(first); - const secondWire = enc.encodeUpdate(second); - expect(firstWire).toMatchObject({ ops: [["s", ["revision"], 1]] }); - expect(secondWire).toMatchObject({ - ops: [ - ["#", 0, ["revision"]], - ["s", 0, 2], - ], - }); - expect(dec.decodeUpdate(firstWire)).toEqual(first); - expect(dec.decodeUpdate(secondWire)).toEqual(second); - }); - - test("validates explicit resets and restarts path dictionaries at the new baseline", () => { - const enc = createServiceStateEncoder(); - const dec = createServiceStateDecoder(); - const snapshot: ServiceSubscriptionSnapshot = { - serviceId: "pi.states", - mode: "singleton", - instances: [{ members: [{ name: "state", kind: "state", sequence: 0, ops: [["r", { before: 0 }]] }] }], - }; - dec.decodeSnapshot(enc.encodeSnapshot(snapshot)); - for (let sequence = 1; sequence <= 2; sequence += 1) { - dec.decodeUpdate( - enc.encodeUpdate({ type: "state", member: "state", sequence, ops: [["s", ["before"], sequence]] }), - ); - } - const reset: ServiceProviderUpdate = { - type: "reset", - snapshot: { - ...snapshot, - instances: [{ members: [{ name: "state", kind: "state", sequence: 103, ops: [["r", { after: 103 }]] }] }], - }, - }; - expect(parseServiceProviderUpdate(reset)).toBe(reset); - const wireReset = enc.encodeUpdate(reset); - expect(parseWireServiceProviderUpdate(wireReset)).toBe(wireReset); - expect(dec.decodeUpdate(wireReset)).toEqual(reset); - for (let sequence = 104; sequence <= 106; sequence += 1) { - const update: ServiceProviderUpdate = { - type: "state", - member: "state", - sequence, - ops: [["s", ["after"], sequence]], - }; - expect(dec.decodeUpdate(enc.encodeUpdate(update))).toEqual(update); - } - for (const parse of [parseServiceProviderUpdate, parseWireServiceProviderUpdate]) { - expect(() => parse({ ...reset, extra: true })).toThrow(); - expect(() => parse({ type: "reset", snapshot: {} })).toThrow(); - expect(() => - parse({ - type: "reset", - snapshot: { - ...snapshot, - instances: [ - { members: [{ name: "state", kind: "state", sequence: 103, ops: [["s", ["before"], 103]] }] }, - ], - }, - }), - ).toThrow(/full root replacements/); - } - }); - - test("isolates operation dictionaries between states and subscriptions", () => { - const snapshot: ServiceSubscriptionSnapshot = { - serviceId: "pi.states", - mode: "singleton", - instances: [ - { - members: [ - { name: "left", kind: "state", sequence: 0, ops: [["r", { revision: 0 }]] }, - { name: "right", kind: "state", sequence: 0, ops: [["r", { revision: 0 }]] }, - ], - }, - ], - }; - const firstEncoder = createServiceStateEncoder(); - const firstDecoder = createServiceStateDecoder(); - const secondEncoder = createServiceStateEncoder(); - const secondDecoder = createServiceStateDecoder(); - firstDecoder.decodeSnapshot(firstEncoder.encodeSnapshot(snapshot)); - secondDecoder.decodeSnapshot(secondEncoder.encodeSnapshot(snapshot)); - - const update = (member: string, sequence: number, revision: number): ServiceProviderUpdate => ({ - type: "state", - member, - sequence, - ops: [["s", ["revision"], revision]], - }); - const firstLeft = firstEncoder.encodeUpdate(update("left", 1, 1)); - const firstRight = firstEncoder.encodeUpdate(update("right", 1, 1)); - const secondLeft = firstEncoder.encodeUpdate(update("left", 2, 2)); - const secondRight = firstEncoder.encodeUpdate(update("right", 2, 2)); - expect(firstLeft).toMatchObject({ ops: [["s", ["revision"], 1]] }); - expect(firstRight).toMatchObject({ ops: [["s", ["revision"], 1]] }); - expect(secondLeft).toMatchObject({ - ops: [ - ["#", 0, ["revision"]], - ["s", 0, 2], - ], - }); - expect(secondRight).toMatchObject({ - ops: [ - ["#", 0, ["revision"]], - ["s", 0, 2], - ], - }); - expect(firstDecoder.decodeUpdate(firstLeft)).toEqual(update("left", 1, 1)); - expect(firstDecoder.decodeUpdate(firstRight)).toEqual(update("right", 1, 1)); - expect(firstDecoder.decodeUpdate(secondLeft)).toEqual(update("left", 2, 2)); - expect(firstDecoder.decodeUpdate(secondRight)).toEqual(update("right", 2, 2)); - - const independentLeft = secondEncoder.encodeUpdate(update("left", 1, 1)); - expect(independentLeft).toMatchObject({ ops: [["s", ["revision"], 1]] }); - expect(secondDecoder.decodeUpdate(independentLeft)).toEqual(update("left", 1, 1)); - - const leftBase: ServiceProviderUpdate = { - type: "state", - member: "left", - sequence: 3, - ops: [["r", { revision: 3 }]], - }; - expect(firstDecoder.decodeUpdate(firstEncoder.encodeUpdate(leftBase))).toEqual(leftBase); - const thirdRight = firstEncoder.encodeUpdate(update("right", 3, 3)); - expect(thirdRight).toMatchObject({ ops: [["s", 0, 3]] }); - expect(firstDecoder.decodeUpdate(thirdRight)).toEqual(update("right", 3, 3)); - }); - - test("creates and removes keyed instance codecs with their lifecycle", () => { - const enc = createServiceStateEncoder(); - const dec = createServiceStateDecoder(); - const snapshot: ServiceSubscriptionSnapshot = { - serviceId: "pi.dialogs", - mode: "keyed", - instances: [], - }; - expect(dec.decodeSnapshot(enc.encodeSnapshot(snapshot))).toEqual(snapshot); - const address = { key: "dialog-1", generation: 1 }; - const spawned: ServiceProviderUpdate = { - type: "spawned", - instance: { - instance: address, - members: [{ name: "request", kind: "state", sequence: 0, ops: [["r", { value: 0 }]] }], - }, - }; - expect(dec.decodeUpdate(enc.encodeUpdate(spawned))).toEqual(spawned); - const update: ServiceProviderUpdate = { - type: "state", - instance: address, - member: "request", - sequence: 1, - ops: [["s", ["value"], 1]], - }; - expect(dec.decodeUpdate(enc.encodeUpdate(update))).toEqual(update); - const closed: ServiceProviderUpdate = { type: "closed", instance: address }; - expect(dec.decodeUpdate(enc.encodeUpdate(closed))).toEqual(closed); - expect(() => enc.encodeUpdate({ ...update, sequence: 2 })).toThrow("Unknown service state"); - }); -}); - -interface Counter { - readonly state: ReplicatedState<{ value: number }>; -} - -const Counter = defineService("test.counter"); - -test("remote service endpoints publish and clean up provider subscriptions", async () => { - const provider = new RemoteServiceProvider([Counter]); - const state = replicatedState({ value: 0 }); - provider.provide(Counter, { state }); - const endpoint = createRemoteServiceEndpoint(provider); - const updates: ServiceProviderUpdate[] = []; - const publish = (_subscriptionId: string, update: ServiceProviderUpdate): void => { - updates.push(update); - }; - - await expect(endpoint.invoke(createServiceCatalogueCall(), publish, BACKGROUND_CONTEXT)).resolves.toEqual([ - { serviceId: Counter.id, mode: "singleton" }, - ]); - await expect( - endpoint.invoke( - createServiceSubscribeCall("subscription-1", Counter.id, "singleton"), - publish, - BACKGROUND_CONTEXT, - ), - ).resolves.toMatchObject({ serviceId: Counter.id, mode: "singleton" }); - - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 1; - }); - expect(updates).toEqual([{ type: "state", member: "state", sequence: 1, ops: [["s", ["value"], 1]] }]); - endpoint.dispose(); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 2; - }); - expect(updates).toHaveLength(1); - provider.dispose(); -}); diff --git a/packages/chord/test/services.test.ts b/packages/chord/test/services.test.ts deleted file mode 100644 index 51bc258f7..000000000 --- a/packages/chord/test/services.test.ts +++ /dev/null @@ -1,790 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; -import { - type Context, - createRemoteServiceBinding, - defineService, - type JsonValue, - RemoteServiceProvider, - type RemoteServiceTransport, - type ReplicatedState, - type ReplicatedStateSource, - type ReplicatedStateSourceFrame, - replicatedState, - type ServiceProviderUpdate, -} from "../src/index.ts"; -import { createLoopbackServiceTransport } from "./helpers.ts"; - -type ModelRef = { provider: string; modelId: string }; -type ModelsState = { - selected: ModelRef | null; - revision: number; -}; -interface Models { - readonly state: ReplicatedState; - select(model: ModelRef, context: Context): Promise; -} - -const Models = defineService("test.models"); - -type Question = { question: string }; -interface QuestionDialogs { - readonly request: ReplicatedState; - submit(answer: string, context: Context): Promise<{ accepted: boolean }>; -} - -const QuestionDialogs = defineService("test.question-dialog"); - -type EchoPayload = { value: string }; -interface Echo { - echo(payload: EchoPayload, context: Context): Promise; -} - -const Echo = defineService("test.echo"); - -type TimelineState = { entries: { id: string }[]; retained: { value: number } }; -interface Timeline { - readonly state: ReplicatedState; -} - -const Timeline = defineService("test.timeline"); - -interface JsonPassthrough { - call(value: JsonValue, context: Context): Promise; -} - -interface NonJsonArgument { - call(value: Date, context: Context): Promise; -} - -interface NonJsonResult { - call(context: Context): Promise; -} - -interface NonJsonState { - readonly state: ReplicatedState<{ value: undefined }>; -} - -describe("remote services", () => { - test("checks remote JSON contracts only at compile time", () => { - expect(defineService("test.json-passthrough").local).toBe(false); - const defineInvalidContracts = (): void => { - // @ts-expect-error Remote service arguments must be JSON-compatible. - defineService("test.non-json-argument"); - // @ts-expect-error Remote service results must be JSON-compatible. - defineService("test.non-json-result"); - // @ts-expect-error Replicated state must be JSON-compatible. - defineService("test.non-json-state"); - }; - expect(defineInvalidContracts).not.toThrow(); - expect(defineService("test.local-non-json", { local: true }).local).toBe(true); - }); - - test("marks services remotable by default and reserves Chord service IDs", () => { - const local = defineService<{ readonly value: string }>("test.local", { local: true }); - expect(Models.local).toBe(false); - expect(local.local).toBe(true); - expect(() => defineService("$chord.internal", { local: true })).toThrow( - "Service IDs beginning with $chord. are reserved", - ); - expect(() => new RemoteServiceProvider([local])).toThrow("cannot be published remotely"); - }); - - test("tracks mutable source state while publishing immutable revisions", () => { - const initial: ModelsState = { selected: null, revision: 0 }; - const state = replicatedState(initial); - let delivered: ModelsState | undefined; - const deliveries: string[] = []; - const unsubscribe = state.subscribe((value, _context, delivery) => { - delivered = value; - deliveries.push(delivery.kind); - }); - expect(state.value).toBe(initial); - expect(delivered).toBe(state.value); - const hydrated = delivered; - - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.selected = { provider: "test", modelId: "one" }; - draft.revision = 1; - }); - expect(state.value).toEqual({ selected: { provider: "test", modelId: "one" }, revision: 1 }); - expect(state.value).not.toBe(initial); - expect(delivered).toBe(state.value); - expect(hydrated).toEqual({ selected: null, revision: 0 }); - expect(deliveries).toEqual(["hydrate", "update"]); - unsubscribe(); - }); - - test("does not publish when a transaction restores the prior value", () => { - const state = replicatedState({ value: 1 }); - const deliveries: Array<{ kind: string; sequence: number }> = []; - state.subscribe((_value, _context, delivery) => deliveries.push(delivery)); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 2; - draft.value = 1; - }); - expect(state.value).toEqual({ value: 1 }); - expect(deliveries).toEqual([{ kind: "hydrate", sequence: 0 }]); - }); - - test("hydrates a new subscriber from the latest atomic revision", () => { - const state = replicatedState({ entries: [{ id: "one" }] }); - const first: { entries: { id: string }[] }[] = []; - state.subscribe((value) => first.push(value)); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.entries.push({ id: "two" }); - }); - - const second: { entries: { id: string }[] }[] = []; - state.subscribe((value) => second.push(value)); - - expect(first).toEqual([{ entries: [{ id: "one" }] }, { entries: [{ id: "one" }, { id: "two" }] }]); - expect(second).toEqual([{ entries: [{ id: "one" }, { id: "two" }] }]); - }); - - test("does not defensively clone method arguments or results", async () => { - const provider = new RemoteServiceProvider([Echo]); - let received: EchoPayload | undefined; - const response: EchoPayload = { value: "response" }; - provider.provide(Echo, { - async echo(payload) { - received = payload; - return response; - }, - }); - const namespace = createRemoteServiceBinding({ - services: [Echo], - transport: createLoopbackServiceTransport(provider), - }); - const echo = namespace.use(Echo); - await namespace.ready(BACKGROUND_CONTEXT); - const request: EchoPayload = { value: "request" }; - - await expect(echo.echo(request, BACKGROUND_CONTEXT)).resolves.toBe(response); - expect(received).toBe(request); - - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("provides and consumes one singleton with replicated state", async () => { - const provider = new RemoteServiceProvider([Models]); - expect(provider.catalogue).toEqual([{ serviceId: Models.id, mode: "singleton" }]); - const initialState: ModelsState = { selected: null, revision: 0 }; - const state = replicatedState(initialState); - let publishedState: ModelsState | undefined; - provider.provide(Models, { - state, - async select(model, context) { - state.change(context, (draft) => { - draft.selected = model; - draft.revision += 1; - }); - publishedState = state.value; - }, - }); - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - onError: (error) => errors.push(error), - }); - - const first = namespace.use(Models); - const second = namespace.use(Models); - expect(first).toBe(second); - expect(first.state.value).toBeUndefined(); - await namespace.ready(BACKGROUND_CONTEXT); - expect(first.state.value).toEqual(initialState); - - const updates: ModelsState[] = []; - const unsubscribe = second.state.subscribe((value) => updates.push(value)); - await first.select({ provider: "test", modelId: "one" }, BACKGROUND_CONTEXT); - expect(first.state.value).toEqual(publishedState); - expect(first.state.value).toEqual({ - selected: { provider: "test", modelId: "one" }, - revision: 1, - }); - expect(updates).toEqual([ - { selected: null, revision: 0 }, - { selected: { provider: "test", modelId: "one" }, revision: 1 }, - ]); - expect(errors).toEqual([]); - - const lateNamespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - }); - const lateModels = lateNamespace.use(Models); - await lateNamespace.ready(BACKGROUND_CONTEXT); - expect(lateModels.state.value?.revision).toBe(1); - - unsubscribe(); - await Promise.all([namespace.dispose(BACKGROUND_CONTEXT), lateNamespace.dispose(BACKGROUND_CONTEXT)]); - provider.dispose(); - }); - - test("publishes compact tracked operations through the remote provider", async () => { - const provider = new RemoteServiceProvider([Timeline]); - const initial = { entries: [{ id: "one" }], retained: { value: 1 } }; - const source = replicatedState(initial); - provider.provide(Timeline, { state: source }); - const updates: Array[2]>[0]> = []; - const raw = provider.subscribe(Timeline.id, "singleton", (update) => updates.push(update)); - expect(raw.snapshot.instances[0]?.members).toEqual([ - { name: "state", kind: "state", sequence: 0, ops: [["r", initial]] }, - ]); - raw.activate(); - - const namespace = createRemoteServiceBinding({ - services: [Timeline], - transport: createLoopbackServiceTransport(provider), - }); - const timeline = namespace.use(Timeline); - await namespace.ready(BACKGROUND_CONTEXT); - const previous = timeline.state.value; - const next = { entries: [{ id: "one" }, { id: "two" }], retained: initial.retained }; - source.change(BACKGROUND_CONTEXT, (draft) => { - draft.entries.push({ id: "two" }); - }); - - expect(updates).toContainEqual({ - type: "state", - member: "state", - sequence: 1, - ops: [["p", ["entries"], 1, 0, [{ id: "two" }]]], - }); - expect(previous).toEqual({ entries: [{ id: "one" }], retained: { value: 1 } }); - expect(timeline.state.value).toEqual(next); - - source.change(BACKGROUND_CONTEXT, (draft) => { - draft.entries.push({ id: "three" }); - }); - const late = provider.subscribe(Timeline.id, "singleton", () => {}); - expect(updates.at(-1)).toEqual({ - type: "state", - member: "state", - sequence: 2, - ops: [["p", ["entries"], 2, 0, [{ id: "three" }]]], - }); - expect(late.snapshot.instances[0]?.members).toEqual([ - { - name: "state", - kind: "state", - sequence: 2, - ops: [["r", { entries: [{ id: "one" }, { id: "two" }, { id: "three" }], retained: { value: 1 } }]], - }, - ]); - late.close(); - raw.close(); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("publishes authoritative source references through services without re-diffing", () => { - const initial: TimelineState = Object.freeze({ entries: [{ id: "one" }], retained: { value: 1 } }); - let publish: ((frame: ReplicatedStateSourceFrame) => void) | undefined; - let disposed = false; - const source: ReplicatedStateSource = { - attach() { - return { - snapshot: { value: initial, cursor: 20 }, - activate(listener) { - publish = listener; - }, - dispose() { - disposed = true; - }, - }; - }, - }; - const state = replicatedState(source); - const provider = new RemoteServiceProvider([Timeline]); - provider.provide(Timeline, { state }); - const updates: ServiceProviderUpdate[] = []; - const subscription = provider.subscribe(Timeline.id, "singleton", (update) => updates.push(update)); - const snapshotMember = subscription.snapshot.instances[0]?.members[0]; - expect(snapshotMember?.kind).toBe("state"); - if (snapshotMember?.kind === "state") expect(snapshotMember.ops[0]?.[1]).toBe(initial); - subscription.activate(); - - const next: TimelineState = Object.freeze({ - entries: [{ id: "one" }, { id: "two" }], - retained: initial.retained, - }); - const ops: ReplicatedStateSourceFrame["ops"] = Object.freeze([ - ["p", ["entries"], 1, 0, [{ id: "two" }]], - ]); - publish?.({ cursor: 21, value: next, ops, context: BACKGROUND_CONTEXT }); - expect(updates).toHaveLength(1); - const update = updates[0]; - expect(update?.type).toBe("state"); - if (update?.type === "state") expect(update.ops).toBe(ops); - - subscription.close(); - provider.dispose(); - state.dispose(); - expect(disposed).toBe(true); - }); - - test("keeps singleton facades stable when their provider is replaced", async () => { - const provider = new RemoteServiceProvider([Models]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 1 }), - async select() {}, - }); - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - }); - const models = namespace.use(Models); - const state = models.state; - const select = models.select; - await namespace.ready(BACKGROUND_CONTEXT); - expect(state.value?.revision).toBe(1); - - provider.withdraw(Models); - expect(state.value).toBeUndefined(); - await expect(select({ provider: "test", modelId: "unavailable" }, BACKGROUND_CONTEXT)).rejects.toMatchObject({ - code: "service_not_found", - }); - - const replacementSelect = vi.fn(async () => {}); - provider.replace(Models, { - state: replicatedState({ selected: null, revision: 2 }), - select: replacementSelect, - }); - - expect(namespace.use(Models)).toBe(models); - expect(models.state).toBe(state); - expect(state.value?.revision).toBe(2); - await select({ provider: "test", modelId: "replacement" }, BACKGROUND_CONTEXT); - expect(replacementSelect).toHaveBeenCalledOnce(); - expect(() => - provider.replace(Models, { - async select() {}, - } as unknown as Models), - ).toThrow("replacement must preserve its member shape"); - expect(() => - provider.replace(Models, { - async state() {}, - async select() {}, - } as unknown as Models), - ).toThrow("replacement must preserve its member shape"); - expect(state.value?.revision).toBe(2); - - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("delivers active subscriber updates before reporting listener failures", () => { - const failure = new Error("listener failed"); - const provider = new RemoteServiceProvider([Models]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 1 }), - async select() {}, - }); - let delivered = 0; - const failing = provider.subscribe(Models.id, "singleton", () => { - throw failure; - }); - const succeeding = provider.subscribe(Models.id, "singleton", () => { - delivered += 1; - }); - failing.activate(); - succeeding.activate(); - - expect(() => - provider.replace(Models, { - state: replicatedState({ selected: null, revision: 2 }), - async select() {}, - }), - ).toThrow(failure); - expect(delivered).toBe(1); - - failing.close(); - succeeding.close(); - provider.dispose(); - }); - - test("does not replay a queued revision already covered by a new subscription snapshot", () => { - const provider = new RemoteServiceProvider([Models]); - const state = replicatedState({ selected: null, revision: 0 }); - provider.provide(Models, { state, async select() {} }); - const lateUpdates: number[] = []; - let late: ReturnType | undefined; - const first = provider.subscribe(Models.id, "singleton", (update) => { - if (update.type !== "state" || update.sequence !== 1) return; - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 2; - }); - late = provider.subscribe(Models.id, "singleton", (next) => { - if (next.type === "state") lateUpdates.push(next.sequence); - }); - expect(late.snapshot.instances[0]?.members).toContainEqual({ - name: "state", - kind: "state", - sequence: 2, - ops: [["r", { selected: null, revision: 2 }]], - }); - late.activate(); - }); - first.activate(); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 1; - }); - expect(lateUpdates).toEqual([]); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 3; - }); - expect(lateUpdates).toEqual([3]); - late?.close(); - first.close(); - provider.dispose(); - }); - - test("replays every buffered update before reporting listener failures", () => { - const failure = new Error("listener failed"); - const provider = new RemoteServiceProvider([Models]); - const state = replicatedState({ selected: null, revision: 0 }); - provider.provide(Models, { state, async select() {} }); - let delivered = 0; - const subscription = provider.subscribe(Models.id, "singleton", () => { - delivered += 1; - throw failure; - }); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 1; - }); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 2; - }); - - expect(() => subscription.activate()).toThrow("Failed to activate remote service subscription"); - expect(delivered).toBe(2); - subscription.close(); - provider.dispose(); - }); - - test("clears retained facades when providers and bindings are disposed", async () => { - const provider = new RemoteServiceProvider([Models]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 1 }), - async select() {}, - }); - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - }); - const models = namespace.use(Models); - const state = models.state; - await namespace.ready(BACKGROUND_CONTEXT); - expect(state.value?.revision).toBe(1); - - provider.dispose(); - expect(state.value).toBeUndefined(); - await expect(models.select({ provider: "test", modelId: "one" }, BACKGROUND_CONTEXT)).rejects.toThrow( - "Remote service provider is disposed", - ); - - await namespace.dispose(BACKGROUND_CONTEXT); - expect(() => state.value).toThrow("Remote service binding is disposed"); - }); - - test("applies provider disposal buffered while subscriptions are starting", async () => { - const provider = new RemoteServiceProvider([Models, { service: QuestionDialogs, mode: "keyed" }]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 1 }), - async select() {}, - }); - provider.spawn(QuestionDialogs, "pending", { - request: replicatedState({ question: "Pending?" }), - async submit() { - return { accepted: true }; - }, - }); - const namespace = createRemoteServiceBinding({ - services: [Models, QuestionDialogs], - transport: createLoopbackServiceTransport(provider), - }); - const models = namespace.use(Models); - const observed: QuestionDialogs[] = []; - namespace.observe(QuestionDialogs, (service) => { - observed.push(service); - }); - - provider.dispose(); - await namespace.ready(BACKGROUND_CONTEXT); - expect(models.state.value).toBeUndefined(); - expect(observed).toEqual([]); - - await namespace.dispose(BACKGROUND_CONTEXT); - }); - - test("keeps deferred service handles inaccessible until host activation", async () => { - const provider = new RemoteServiceProvider([Models]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 0 }), - async select() {}, - }); - let active = false; - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - bound: false, - assertAccess() { - if (!active) throw new Error("Service handles are not active"); - }, - }); - const models = namespace.use(Models); - - expect(() => models.state.value).toThrow("Service handles are not active"); - expect(() => models.state.subscribe(() => {})).toThrow("Service handles are not active"); - expect(() => models.select({ provider: "test", modelId: "one" }, BACKGROUND_CONTEXT)).toThrow( - "Service handles are not active", - ); - - await namespace.rebind(true, BACKGROUND_CONTEXT); - active = true; - await namespace.ready(BACKGROUND_CONTEXT); - expect(models.state.value?.revision).toBe(0); - - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("rejects namespace readiness when initial hydration fails", async () => { - const failure = new Error("initial hydration failed"); - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: { - invoke: () => Promise.reject(new Error("unexpected invocation")), - subscribe: () => Promise.reject(failure), - }, - onError: (error) => errors.push(error), - }); - const models = namespace.use(Models); - - await expect(namespace.ready(BACKGROUND_CONTEXT)).rejects.toBe(failure); - expect(models.state.value).toBeUndefined(); - expect(errors).toEqual([failure]); - await namespace.dispose(BACKGROUND_CONTEXT); - }); - - test("buffers state updates that race subscription hydration", async () => { - const provider = new RemoteServiceProvider([Models]); - const state = replicatedState({ selected: null, revision: 0 }); - provider.provide(Models, { - state, - async select() {}, - }); - const transport: RemoteServiceTransport = { - invoke: (call, context) => provider.invoke(call, context), - subscribe: async (serviceId, mode, listener) => { - const subscription = provider.subscribe(serviceId, mode, listener); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 1; - }); - return { - snapshot: subscription.snapshot, - activate: () => subscription.activate(), - close: () => subscription.close(), - }; - }, - }; - const namespace = createRemoteServiceBinding({ services: [Models], transport }); - const models = namespace.use(Models); - const revisions: number[] = []; - models.state.subscribe((value) => revisions.push(value.revision)); - - await vi.waitFor(() => expect(revisions).toEqual([0, 1])); - expect(models.state.value?.revision).toBe(1); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test.each([ - ["duplicate", 0], - ["gap", 2], - ] as const)("clears replicated state after a %s operation sequence", async (_kind, sequence) => { - let sendUpdate: ((update: Parameters[2]>[0]) => void) | undefined; - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: { - invoke: () => Promise.reject(new Error("unexpected invocation")), - async subscribe(_serviceId, _mode, listener) { - sendUpdate = (update) => listener(update, BACKGROUND_CONTEXT); - return { - snapshot: { - serviceId: Models.id, - mode: "singleton", - instances: [ - { - members: [ - { name: "select", kind: "method" }, - { - name: "state", - kind: "state", - sequence: 0, - ops: [["r", { selected: null, revision: 0 }]], - }, - ], - }, - ], - }, - activate() {}, - close() {}, - }; - }, - }, - onError: (error) => errors.push(error), - }); - const models = namespace.use(Models); - await namespace.ready(BACKGROUND_CONTEXT); - expect(models.state.value?.revision).toBe(0); - - sendUpdate?.({ - type: "state", - member: "state", - sequence, - ops: [["r", { selected: null, revision: sequence }]], - }); - expect(models.state.value).toBeUndefined(); - expect(errors).toHaveLength(1); - expect(errors[0]?.message).toContain("sequence has a gap"); - await namespace.dispose(BACKGROUND_CONTEXT); - }); - - test("hydrates cold ReplicatedState replicas and replaces them across rebinds", async () => { - const provider = new RemoteServiceProvider([Models]); - const state = replicatedState({ selected: null, revision: 0 }); - provider.provide(Models, { - state, - async select() {}, - }); - const namespace = createRemoteServiceBinding({ - services: [Models], - transport: createLoopbackServiceTransport(provider), - bound: false, - }); - const models = namespace.use(Models); - const revisions: number[] = []; - models.state.subscribe((value) => revisions.push(value.revision)); - expect(models.state.value).toBeUndefined(); - expect(revisions).toEqual([]); - - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 1; - }); - await namespace.rebind(true, BACKGROUND_CONTEXT); - expect(models.state.value?.revision).toBe(1); - expect(revisions).toEqual([1]); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 2; - }); - expect(revisions).toEqual([1, 2]); - - await namespace.rebind(false, BACKGROUND_CONTEXT); - expect(models.state.value).toBeUndefined(); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.revision = 3; - }); - expect(revisions).toEqual([1, 2]); - await namespace.rebind(true, BACKGROUND_CONTEXT); - expect(models.state.value?.revision).toBe(3); - expect(revisions).toEqual([1, 2, 3]); - - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("hydrates keyed state before observe handlers and fences reused keys", async () => { - const provider = new RemoteServiceProvider([{ service: QuestionDialogs, mode: "keyed" }]); - expect(provider.catalogue).toEqual([{ serviceId: QuestionDialogs.id, mode: "keyed" }]); - const transport = createLoopbackServiceTransport(provider); - const errors: Error[] = []; - const namespace = createRemoteServiceBinding({ - services: [QuestionDialogs], - transport, - onError: (error) => errors.push(error), - }); - const observed: { - question: Question | undefined; - service: QuestionDialogs; - context: Context; - }[] = []; - const stop = namespace.observe(QuestionDialogs, (service, context) => { - observed.push({ - question: service.request.value, - service, - context, - }); - }); - await vi.waitFor(() => expect(errors).toEqual([])); - - const firstRequest = replicatedState({ question: "First?" }); - const firstSubmit = vi.fn(async () => ({ accepted: true })); - const closeFirst = provider.spawn(QuestionDialogs, "invocation-1", { - request: firstRequest, - submit: firstSubmit, - }); - await vi.waitFor(() => expect(observed).toHaveLength(1)); - expect(observed[0]).toMatchObject({ question: { question: "First?" } }); - - const firstService = observed[0]!.service; - firstRequest.change(BACKGROUND_CONTEXT, (draft) => { - draft.question = "Updated?"; - }); - expect(firstService.request.value).toEqual({ question: "Updated?" }); - await expect(firstService.submit("yes", BACKGROUND_CONTEXT)).resolves.toEqual({ accepted: true }); - expect(firstSubmit).toHaveBeenCalledWith("yes", expect.objectContaining({ abortSignal: undefined })); - - const retainedFirstSubmit = firstService.submit; - closeFirst(); - expect(observed[0]!.context.abortSignal?.aborted).toBe(true); - expect(() => firstService.request.value).toThrow("observation is closed"); - expect(() => retainedFirstSubmit("late", BACKGROUND_CONTEXT)).toThrow("observation is closed"); - - const secondRequest = replicatedState({ question: "Again?" }); - const closeSecond = provider.spawn(QuestionDialogs, "invocation-1", { - request: secondRequest, - async submit() { - return { accepted: false }; - }, - }); - await vi.waitFor(() => expect(observed).toHaveLength(2)); - expect(observed[1]).toMatchObject({ question: { question: "Again?" } }); - expect(Object.is(observed[1]!.service, firstService)).toBe(false); - expect(errors).toEqual([]); - - const secondService = observed[1]!.service; - const retainedSecondSubmit = secondService.submit; - stop(); - expect(observed[1]!.context.abortSignal?.aborted).toBe(true); - expect(() => secondService.request.value).toThrow("observation is closed"); - expect(() => retainedSecondSubmit("late", BACKGROUND_CONTEXT)).toThrow("observation is closed"); - closeSecond(); - await namespace.dispose(BACKGROUND_CONTEXT); - provider.dispose(); - }); - - test("rejects mode mixing and unsupported members", () => { - const provider = new RemoteServiceProvider([Models, { service: QuestionDialogs, mode: "keyed" }]); - provider.provide(Models, { - state: replicatedState({ selected: null, revision: 0 }), - async select() {}, - }); - expect(() => provider.spawn(Models, "wrong", {} as Models)).toThrow(/singleton/); - expect(() => - provider.spawn(QuestionDialogs, "invalid", { - request: new Date() as unknown as ReplicatedState, - async submit() { - return { accepted: true }; - }, - }), - ).toThrow(/not remotely exposable/); - provider.dispose(); - }); -}); diff --git a/packages/chord/test/state-diff.test.ts b/packages/chord/test/state-diff.test.ts index 6d07512c5..22b9d1229 100644 --- a/packages/chord/test/state-diff.test.ts +++ b/packages/chord/test/state-diff.test.ts @@ -8,7 +8,7 @@ import { encoder, type Op, } from "../src/delta/index.ts"; -import type { JsonValue } from "../src/types.ts"; +import type { JsonValue } from "../src/json.ts"; const expectDiff = (before: JsonValue, after: JsonValue, expected: unknown): void => { const operations = diffRevisions(before, after); diff --git a/packages/chord/test/state.test.ts b/packages/chord/test/state.test.ts index d175db524..3b2cb2887 100644 --- a/packages/chord/test/state.test.ts +++ b/packages/chord/test/state.test.ts @@ -1,411 +1,283 @@ -import { describe, expect, it } from "vitest"; -import { BACKGROUND_CONTEXT } from "../src/context/index.ts"; -import type { Draft, Op } from "../src/delta/index.ts"; +import assert from "node:assert/strict"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { applyImmutable } from "../src/delta/index.ts"; import { - type Context, - type JsonValue, - type ReplicatedStateSource, - type ReplicatedStateSourceAttachment, - type ReplicatedStateSourceFrame, - replicatedState, + type Ctx, + cell, + createHost, + derivation, + isChordError, + listener, + local, + method, + resource, + type StateEvent, } from "../src/index.ts"; -import { ReplicatedStateReplica } from "../src/services/state.ts"; -import { getReplicatedStateInternals } from "../src/services/state-internals.ts"; -describe("transactional replicated state", () => { - it("publishes one immutable structurally shared revision", () => { - const initial = { changed: { value: 1 }, retained: { value: 2 } }; - const state = replicatedState(initial); - const previous = state.value; - const deliveries: number[] = []; - state.subscribe((_value, _context, delivery) => deliveries.push(delivery.sequence)); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.changed.value = 3; - draft.changed.value = 4; - }); - expect(state.value).toEqual({ changed: { value: 4 }, retained: { value: 2 } }); - expect(state.value).not.toBe(previous); - expect(state.value.changed).not.toBe(previous.changed); - expect(state.value.retained).toBe(previous.retained); - expect(Object.isFrozen(state.value)).toBe(false); - expect(deliveries).toEqual([0, 1]); - }); +const tick = () => new Promise((resolve) => setImmediate(resolve)); - it("rolls back callback failures and revokes escaped drafts", () => { - const state = replicatedState({ nested: { value: 1 } }); - const previous = state.value; - let escaped: Draft<{ value: number }> | undefined; - expect(() => - state.change(BACKGROUND_CONTEXT, (draft) => { - escaped = draft.nested; - draft.nested.value = 2; - throw new Error("stop"); - }), - ).toThrow("stop"); - expect(state.value).toBe(previous); - expect(() => escaped?.value).toThrow(TypeError); - }); +function emitter() { + const listeners = new Set<(event: E) => void>(); + return { + emit: (event: E) => { + for (const fn of listeners) fn(event); + }, + subscribe: (fn: (event: E) => void) => { + listeners.add(fn); + return () => listeners.delete(fn); + }, + get size() { + return listeners.size; + }, + }; +} - it("rejects PromiseLike callbacks and aborts their draft", () => { - const state = replicatedState({ value: 0 }); - const previous = state.value; - expect(() => - state.change(BACKGROUND_CONTEXT, ((draft: Draft<{ value: number }>) => { - draft.value = 1; - return Promise.resolve(); - }) as (draft: Draft<{ value: number }>) => void), - ).toThrow(/synchronous/); - expect(state.value).toBe(previous); - }); +const Item = Type.Object({ id: Type.Number(), label: Type.String() }); +const Items = Type.Object({ items: Type.Array(Item), count: Type.Number() }); - it("rejects nested changes and replacements without losing the outer rollback", () => { - const state = replicatedState({ left: 0, right: 0 }); - expect(() => - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.left = 1; - state.change(BACKGROUND_CONTEXT, (nested) => { - nested.right = 2; - }); - }), - ).toThrow(/reentrantly/); - expect(state.value).toEqual({ left: 0, right: 0 }); - expect(() => - state.change(BACKGROUND_CONTEXT, () => { - state.replace(BACKGROUND_CONTEXT, { left: 1, right: 2 }); - }), - ).toThrow(/change callback/); - expect(state.value).toEqual({ left: 0, right: 0 }); - }); - - it("queues listener-triggered changes in sequence order", () => { - const state = replicatedState({ value: 0 }); - const sourceSequences: number[] = []; - const deliveries: Array<{ value: number; sequence: number }> = []; - const lateDeliveries: Array<{ kind: string; sequence: number }> = []; - let nested = false; - getReplicatedStateInternals(state)!.subscribe((_operations, sequence) => { - if (!nested) { - nested = true; - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 2; - }); - state.subscribe((_value, _context, delivery) => lateDeliveries.push(delivery)); - } - void sequence; - }); - getReplicatedStateInternals(state)!.subscribe((_operations, sequence) => sourceSequences.push(sequence)); - state.subscribe((value, _context, delivery) => { - if (delivery.kind === "update") deliveries.push({ value: value.value, sequence: delivery.sequence }); - }); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 1; - }); - expect(sourceSequences).toEqual([1, 2]); - expect(deliveries).toEqual([ - { value: 1, sequence: 1 }, - { value: 2, sequence: 2 }, - ]); - expect(lateDeliveries).toEqual([{ kind: "hydrate", sequence: 2 }]); - }); - - it("isolates listener failures after committing the revision", () => { - const state = replicatedState({ value: 0 }); - const received: number[] = []; - getReplicatedStateInternals(state)!.subscribe(() => { - throw new Error("listener failed"); - }); - getReplicatedStateInternals(state)!.subscribe((_operations, sequence) => received.push(sequence)); - expect(() => - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.value = 1; - }), - ).toThrow("listener failed"); - expect(state.value).toEqual({ value: 1 }); - expect(received).toEqual([1]); - }); - - it("copies assigned values by value", () => { - const external = { value: 1 }; - const state = replicatedState<{ left: { value: number } | null; right: { value: number } | null }>({ - left: null, - right: null, - }); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.left = external; - draft.right = external; - draft.left.value = 2; - }); - expect(external.value).toBe(1); - expect(state.value).toEqual({ left: { value: 2 }, right: { value: 1 } }); - expect(state.value.left).not.toBe(state.value.right); - }); - - it("takes immutable ownership of alias-free replacements", () => { - const state = replicatedState({ left: { value: 1 }, right: { value: 2 } }); - const replacement = { left: { ...state.value.left }, right: { ...state.value.left } }; - state.replace(BACKGROUND_CONTEXT, replacement); - expect(state.value).toBe(replacement); - expect(state.value.left).not.toBe(state.value.right); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.left.value = 9; - }); - expect(state.value).toEqual({ left: { value: 9 }, right: { value: 1 } }); - }); - - it("preserves compact string, splice, and permutation operations", () => { - const a = { id: "a" }; - const b = { id: "b" }; - const c = { id: "c" }; - const state = replicatedState({ text: "abcdefgh", values: [a, b, c] }); - const batches: JsonValue[] = []; - getReplicatedStateInternals(state)!.subscribe((operations) => batches.push(operations as unknown as JsonValue)); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.text = "defghxyz"; - draft.values.shift(); - }); - state.change(BACKGROUND_CONTEXT, (draft) => { - draft.values.reverse(); - }); - expect(batches).toEqual([ - [ - ["t", ["text"], 3], - ["a", ["text"], "xyz"], - ["p", ["values"], 0, 1, []], - ], - [["m", ["values"], [1, 0]]], - ]); - }); - - it("validates replica revisions without freezing shared immutable payloads", () => { - const replica = new ReplicatedStateReplica<{ rows: { value: number }[] }>(() => {}); - const initial = { rows: [{ value: 1 }] }; - replica.hydrate(0, [["r", initial]], BACKGROUND_CONTEXT); - expect(replica.value).toBe(initial); - expect(Object.isFrozen(initial)).toBe(false); - expect(Object.isFrozen(initial.rows[0])).toBe(false); - - const inserted = { value: 2 }; - replica.update(1, [["p", ["rows"], 1, 0, [inserted]]], BACKGROUND_CONTEXT); - expect(replica.value).not.toBe(initial); - expect(replica.value?.rows).toEqual([{ value: 1 }, { value: 2 }]); - expect(replica.value?.rows[0]).toBe(initial.rows[0]); - expect(replica.value?.rows[1]).toBe(inserted); - expect(initial.rows).toHaveLength(1); - expect(Object.isFrozen(inserted)).toBe(false); - }); - - it("clears a replica when an adopted update is invalid", () => { - const errors: Error[] = []; - const replica = new ReplicatedStateReplica<{ value: number }>((error) => errors.push(error)); - replica.hydrate(0, [["r", { value: 0 }]], BACKGROUND_CONTEXT); - expect(() => replica.update(1, [["s", ["value"], Number.NaN]], BACKGROUND_CONTEXT)).toThrow(/strict JSON/); - expect(replica.value).toBeUndefined(); - expect(() => replica.update(2, [["s", ["value"], 2]], BACKGROUND_CONTEXT)).toThrow(/before hydration/); - - const malformed = new ReplicatedStateReplica<{ values: number[] }>((error) => errors.push(error)); - malformed.hydrate(0, [["r", { values: [1, 2] }]], BACKGROUND_CONTEXT); - expect(() => malformed.update(1, [["m", ["values"], [0]]], BACKGROUND_CONTEXT)).toThrow(); - expect(malformed.value).toBeUndefined(); - expect(errors).toEqual([]); - }); - - it("replaces atomically and ignores deeply equal replacements", () => { - const state = replicatedState({ value: { nested: 1 }, retained: { nested: 2 } }); - const previous = state.value; - state.replace(BACKGROUND_CONTEXT, { value: { nested: 1 }, retained: { nested: 2 } }); - expect(state.value).toBe(previous); - state.replace(BACKGROUND_CONTEXT, { ...state.value, value: { nested: 2 } }); - expect(state.value).toEqual({ value: { nested: 2 }, retained: { nested: 2 } }); - expect(state.value.retained).toBe(previous.retained); - }); +const List = local("t.list", { + version: 1, + input: Type.Object({}), + members: { + list: Items, + labels: Type.Array(Type.String()), + push: method(Item, Type.Null()), + set: method(Items, Type.Null()), + }, }); -type SourceValue = Readonly<{ value: number }>; - -class TestSource implements ReplicatedStateSource { - readonly attachments = new Set>(); - onAttach: (() => void) | undefined; - #value: T; - #cursor: number; - - constructor(value: T, cursor = 0) { - this.#value = value; - this.#cursor = cursor; - } - - attach(): ReplicatedStateSourceAttachment { - const attachment = new TestSourceAttachment(Object.freeze({ value: this.#value, cursor: this.#cursor }), () => - this.attachments.delete(attachment), - ); - this.attachments.add(attachment); - this.onAttach?.(); - return attachment; - } - - commit(value: T, ops: readonly Op[], context: Context = BACKGROUND_CONTEXT, cursor = this.#cursor + 1): void { - this.#value = value; - this.#cursor = cursor; - const frame = Object.freeze({ cursor, value, ops, context }); - for (const attachment of [...this.attachments]) attachment.publish(frame); - } -} - -class TestSourceAttachment implements ReplicatedStateSourceAttachment { - readonly snapshot: { readonly value: T; readonly cursor: number }; - readonly #onDispose: () => void; - readonly #buffer: ReplicatedStateSourceFrame[] = []; - #listener: ((frame: ReplicatedStateSourceFrame) => void) | undefined; - #activated = false; - #disposed = false; - - constructor(snapshot: { readonly value: T; readonly cursor: number }, onDispose: () => void) { - this.snapshot = snapshot; - this.#onDispose = onDispose; - } - - activate(listener: (frame: ReplicatedStateSourceFrame) => void): void { - if (this.#activated) throw new Error("attachment is already active"); - this.#activated = true; - this.#listener = listener; - for (const frame of this.#buffer.splice(0)) listener(frame); - } - - publish(frame: ReplicatedStateSourceFrame): void { - if (this.#disposed) return; - if (this.#listener === undefined) this.#buffer.push(frame); - else this.#listener(frame); - } - - dispose(): void { - if (this.#disposed) return; - this.#disposed = true; - this.#buffer.length = 0; - this.#onDispose(); - } -} - -describe("authoritative replicated state sources", () => { - it("captures before activation and drains queued commits in order", () => { - const initial = Object.freeze({ value: 0 }); - const first = Object.freeze({ value: 1 }); - const second = Object.freeze({ value: 2 }); - const firstOps = Object.freeze([["s", ["value"], 1] as const]); - const secondOps = Object.freeze([["s", ["value"], 2] as const]); - const source = new TestSource(initial, 10); - source.onAttach = () => { - source.commit(first, firstOps); - source.commit(second, secondOps); +function serveList(host: ReturnType) { + host.serve(List, (_input, _principal, self) => { + const state = self.state({ list: cell(Items, { items: [], count: 0 }) }); + const derived = self.derive({ + labels: derivation([state.list], (list) => list.items.map((item) => item.label)), + }); + return { + list: state.list, + labels: derived.labels, + push: (item: { id: number; label: string }, ctx: Ctx) => { + state.list.change(ctx, (draft) => { + draft.items.push(item); + draft.count += 1; + }); + return null; + }, + set: (value: { items: { id: number; label: string }[]; count: number }, ctx: Ctx) => { + state.list.replace(ctx, value); + return null; + }, }; + }); +} - const state = replicatedState(source); - const deliveries: Array<{ value: SourceValue; sequence: number }> = []; - state.subscribe((value, _context, delivery) => deliveries.push({ value, sequence: delivery.sequence })); - expect(state.value).toBe(second); - expect(deliveries).toEqual([{ value: second, sequence: 2 }]); - expect(getReplicatedStateInternals(state)?.snapshot()).toEqual({ value: second, sequence: 2 }); +describe("state cells", () => { + it("publishes snapshots then exact ops; derivations follow", async () => { + const host = createHost({ onError: () => {} }); + serveList(host); + const handle = host.get(List, {}); + const events: StateEvent[] = []; + const labels: StateEvent[] = []; + handle.list.events((event) => events.push(event)); + handle.labels.events((event) => labels.push(event)); + await handle.push({ id: 1, label: "a" }); + // Subscribed while loading: an empty snapshot, the hydrated value as a reset, then exact ops. + assert.deepEqual(events[0], { type: "snap", value: undefined }); + assert.deepEqual(events[1], { type: "reset", value: { items: [], count: 0 } }); + const ops = events[2]?.type === "ops" ? events[2].ops.map((op) => JSON.stringify(op)).sort() : []; + assert.deepEqual(ops, ['["p",["items"],0,0,[{"id":1,"label":"a"}]]', '["s",["count"],1]']); + assert.deepEqual(events[2]?.value, { items: [{ id: 1, label: "a" }], count: 1 }); + assert.deepEqual(handle.labels.value, ["a"]); + assert.deepEqual( + labels.map((event) => event.type), + ["snap", "reset", "ops"], + ); + await handle.set({ items: [], count: 0 }); + assert.deepEqual(handle.labels.value, []); }); - it("hydrates at sequence zero when attaching after existing commits", () => { - const initial = Object.freeze({ value: 0 }); - const current = Object.freeze({ value: 1 }); - const source = new TestSource(initial, 40); - source.commit(current, Object.freeze([["s", ["value"], 1] as const])); - const state = replicatedState(source); - const deliveries: number[] = []; - state.subscribe((_value, _context, delivery) => deliveries.push(delivery.sequence)); - expect(state.value).toBe(current); - expect(deliveries).toEqual([0]); + it("supports primitive cells and value-returning changes", async () => { + const host = createHost({ onError: () => {} }); + const Counter = local("t.prim", { + version: 1, + input: Type.Object({}), + members: { n: Type.Number(), inc: method(Type.Null(), Type.Number()) }, + }); + host.serve(Counter, (_input, _principal, self) => { + const state = self.state({ n: cell(Type.Number(), 0) }); + return { + n: state.n, + inc: (_args: null, ctx: Ctx) => { + state.n.change(ctx, (n) => n + 1); + return state.n.value; + }, + }; + }); + const handle = host.get(Counter, {}); + const values: unknown[] = []; + await handle.ready; + const events: StateEvent[] = []; + handle.n.events((event) => events.push(event)); + handle.n.subscribe((value) => values.push(value)); + assert.equal(await handle.inc(null), 1); + assert.equal(await handle.inc(null), 2); + assert.deepEqual(values, [0, 1, 2]); + assert.deepEqual(events, [ + { type: "snap", value: 0 }, + { type: "ops", ops: [["r", 1]], value: 1 }, + { type: "ops", ops: [["r", 2]], value: 2 }, + ]); }); - it("publishes exact source value and operation references without applying or re-diffing", () => { - const source = new TestSource(Object.freeze({ value: 0 })); - const state = replicatedState(source); - const next = Object.freeze({ value: 1 }); - const ops = Object.freeze([["s", ["value"], 1] as const]); - let publishedOps: readonly Op[] | undefined; - getReplicatedStateInternals(state)?.subscribe((received) => { - publishedOps = received; + it("rejects a returned value that does not fit the schema, even outside dev mode", async () => { + const host = createHost({ onError: () => {} }); + const Box = local("t.box", { + version: 1, + input: Type.Object({}), + members: { put: method(Type.Null(), Type.Null()) }, }); - let publishedValue: SourceValue | undefined; - state.subscribe((value, _context, delivery) => { - if (delivery.kind === "update") publishedValue = value; + host.serve(Box, (_input, _principal, self) => { + const state = self.state({ list: cell(Type.Array(Type.Number()), []) }); + return { + put: (_args: null, ctx: Ctx) => { + // The classic slip: an expression-bodied arrow returns push()'s length. + state.list.change(ctx, (draft) => draft.push(1) as never); + return null; + }, + }; }); - - source.commit(next, ops); - expect(state.value).toBe(next); - expect(publishedValue).toBe(next); - expect(publishedOps).toBe(ops); + await assert.rejects(host.get(Box, {}).put(null), (error) => isChordError(error, "invalid")); }); - it("buffers reentrant frames and skips updates covered by a late hydration", () => { - const source = new TestSource(Object.freeze({ value: 0 })); - const state = replicatedState(source); - const received: number[] = []; - const late: Array<{ kind: string; sequence: number; value: number }> = []; - let nested = false; - getReplicatedStateInternals(state)?.subscribe((_ops, sequence) => { - if (sequence !== 1 || nested) return; - nested = true; - source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); - state.subscribe((value, _context, delivery) => late.push({ ...delivery, value: value.value })); - }); - state.subscribe((value, _context, delivery) => { - if (delivery.kind === "update") received.push(value.value); + it("validates initial values always and writes only in dev mode", async () => { + const Bad = local("t.bad-initial", { version: 1, input: Type.Object({}) }); + const host = createHost({ onError: () => {} }); + host.serve(Bad, (_input, _principal, self) => { + self.state({ x: cell(Type.Object({ n: Type.Number() }), { n: "no" } as never) }); + return {}; }); + await assert.rejects(host.get(Bad, {}).ready, /initial value of state x/); - source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])); - expect(received).toEqual([1, 2]); - expect(late).toEqual([{ kind: "hydrate", sequence: 2, value: 2 }]); + const lax = createHost({ onError: () => {} }); + serveList(lax); + await lax.get(List, {}).set({ items: [], count: "x" } as never); + + const dev = createHost({ dev: true, onError: () => {} }); + serveList(dev); + const handle = dev.get(List, {}); + await assert.rejects( + handle.set({ items: [], count: "x" } as never), + (error) => isChordError(error, "invalid") && error.reason === "state", + ); + assert.deepEqual(handle.list.value, { items: [], count: 0 }); }); - it("reports listener failures without throwing them into the source", () => { - const source = new TestSource(Object.freeze({ value: 0 })); - const errors: Error[] = []; - const state = replicatedState(source, { onError: (error) => errors.push(error) }); - const received: number[] = []; - state.subscribe((_value, _context, delivery) => { - if (delivery.kind === "update") throw new Error("listener failed"); + it("rejects writes from inside a factory and placeholder use before activation", async () => { + const host = createHost({ onError: () => {} }); + const Writer = local("t.writer", { version: 1, input: Type.Object({}) }); + let captured: Ctx | undefined; + await host.root((ctx) => { + captured = ctx; }); - state.subscribe((value, _context, delivery) => { - if (delivery.kind === "update") received.push(value.value); + host.serve(Writer, (_input, _principal, self) => { + const state = self.state({ x: cell(Type.Object({}), {}) }); + const res = self.resources({ r: resource({ rev: 1, open: () => 1 }) }); + assert.throws( + () => res.r.value, + (error) => isChordError(error, "not_ready"), + ); + state.x.replace(captured!, {}); + return {}; }); - - expect(() => - source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])), - ).not.toThrow(); - source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); - expect(received).toEqual([1, 2]); - expect(errors.map((error) => error.message)).toEqual(["listener failed", "listener failed"]); + await assert.rejects(host.get(Writer, {}).ready, /read_only/); }); - it("reports cursor gaps, disposes the broken attachment, and ignores later frames", () => { - const source = new TestSource(Object.freeze({ value: 0 }), 5); - const errors: Error[] = []; - const state = replicatedState(source, { onError: (error) => errors.push(error) }); - source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const]), BACKGROUND_CONTEXT, 7); - expect(errors[0]?.message).toContain("expected 6, received 7"); - expect(source.attachments.size).toBe(0); - expect(state.value).toEqual({ value: 0 }); - source.commit(Object.freeze({ value: 3 }), Object.freeze([["s", ["value"], 3] as const]), BACKGROUND_CONTEXT, 8); - expect(state.value).toEqual({ value: 0 }); + it("listeners run each event in a fresh root; tasks get an abortable ctx", async () => { + const host = createHost({ onError: () => {} }); + const events = emitter(); + const Watcher = local("t.watcher", { + version: 1, + input: Type.Object({}), + members: { seen: Type.Object({ sum: Type.Number() }) }, + }); + const roots = new Set(); + let taskAborted = false; + host.serve(Watcher, (_input, _principal, self) => { + const state = self.state({ seen: cell(Type.Object({ sum: Type.Number() }), { sum: 0 }) }); + const res = self.resources({ source: resource({ rev: 1, open: () => events }) }); + self.listen({ + onEvent: listener( + res.source, + (source) => source, + (value, ctx) => { + roots.add(ctx.rootId); + state.seen.change(ctx, (draft) => { + draft.sum += value; + }); + }, + ), + }); + self.tasks({ + loop: async (ctx) => { + await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); + taskAborted = true; + }, + }); + return { seen: state.seen }; + }); + const handle = host.get(Watcher, {}); + await handle.ready; + events.emit(2); + events.emit(3); + await tick(); + assert.deepEqual(handle.seen.value, { sum: 5 }); + assert.equal(roots.size, 2); + handle.release(); + await tick(); + await tick(); + assert.equal(events.size, 0); + assert.ok(taskAborted); }); - it("keeps attachments independent and disposes each idempotently", () => { - const source = new TestSource(Object.freeze({ value: 0 })); - const first = replicatedState(source); - const second = replicatedState(source); - expect(source.attachments.size).toBe(2); - source.commit(Object.freeze({ value: 1 }), Object.freeze([["s", ["value"], 1] as const])); - expect(first.value.value).toBe(1); - expect(second.value.value).toBe(1); - - first.dispose(); - first.dispose(); - expect(source.attachments.size).toBe(1); - source.commit(Object.freeze({ value: 2 }), Object.freeze([["s", ["value"], 2] as const])); - expect(first.value.value).toBe(1); - expect(second.value.value).toBe(2); - second.dispose(); - expect(source.attachments.size).toBe(0); + it("re-entrant writes publish in adoption order; derivations never go stale (Sol #3)", async () => { + const host = createHost({ onError: () => {} }); + const Echo = local("t.reentrant", { + version: 1, + input: Type.Object({}), + members: { n: Type.Number(), d: Type.Number(), set: method(Type.Number(), Type.Null()) }, + }); + host.serve(Echo, (_input, _principal, self) => { + const state = self.state({ n: cell(Type.Number(), 0) }); + const derived = self.derive({ d: derivation([state.n], (n) => n * 2) }); + // Subscribed first: re-enters the write while revision 1 is being published. + self.listen({ + bump: listener(state.n, (n, ctx) => { + if (n === 1) state.n.replace(ctx, 2); + }), + }); + return { + n: state.n, + d: derived.d, + set: (value: number, ctx: Ctx) => { + state.n.replace(ctx, value); + return null; + }, + }; + }); + const handle = host.get(Echo, {}); + await handle.ready; + const values: unknown[] = []; + let replica: unknown; + handle.n.events((event) => { + replica = event.type === "ops" ? applyImmutable(replica, event.ops) : event.value; + values.push(replica); + }); + const doubled: unknown[] = []; + handle.d.subscribe((value) => doubled.push(value)); + await handle.set(1); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(values, [0, 1, 2]); + assert.equal(handle.d.value, 4); + assert.deepEqual(doubled, [0, 2, 4]); }); }); diff --git a/packages/chord/test/tutorial.test.ts b/packages/chord/test/tutorial.test.ts new file mode 100644 index 000000000..fe465436f --- /dev/null +++ b/packages/chord/test/tutorial.test.ts @@ -0,0 +1,527 @@ +// example:imports +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { type Static, Type } from "typebox"; +import { it, onTestFinished } from "vitest"; +import { + type Ctx, + cell, + createHost, + defineFacet, + defineRegistry, + derivation, + type Facet, + type FacetApi, + type Factory, + forbidden, + type Host, + hostPrincipal, + invalid, + isChordError, + listener, + local, + memoryLinkPair, + method, + opaque, + remote, + resource, +} from "../src/index.ts"; + +// end-example:imports + +// Compare every printed example with its runnable source, not a second handwritten copy. +const tutorial = readFileSync(new URL("../docs/tutorial.md", import.meta.url), "utf8"); +const source = readFileSync(new URL(import.meta.url), "utf8"); +const examples = [...source.matchAll(/^[\t ]*\/\/ example:([\w-]+)\n([\s\S]*?)^[\t ]*\/\/ end-example:\1$/gm)]; +const printed = [...tutorial.matchAll(/\n```ts\n([\s\S]*?)\n```/g)]; +function normalize(text: string): string { + return text + .split("\n") + .map((line) => line.trim()) + .join("\n") + .trim(); +} +function checkExamples(section: string): void { + assert.equal(printed.length, examples.length, "every tutorial example must have runnable source"); + assert.equal((tutorial.match(/```ts\n/g) ?? []).length, printed.length, "no untracked TypeScript examples"); + for (const match of examples.filter((entry) => entry[1] === "imports" || entry[1]?.startsWith(section))) { + const copies = printed.filter((entry) => entry[1] === match[1]); + assert.equal(copies.length, 1, `one printed copy of ${match[1]}`); + assert.equal(normalize(copies[0]![2]!), normalize(match[2]!), `tutorial drift: ${match[1]}`); + } +} + +// example:02-contract +const TodoLabel = local("tutorial.todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { label: method(Type.Null(), Type.String()) }, +}); +// end-example:02-contract + +// example:03-contract +const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); +const Rows = Type.Array(Todo); +const TodoStore = local("tutorial.todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { + label: method(Type.Null(), Type.String()), + rows: Rows, + openCount: Type.Number(), + add: method(Type.Object({ text: Type.String() }), Type.Null()), + finish: method(Type.Object({ index: Type.Integer() }), Type.Null()), + reset: method(Rows, Type.Null()), + }, +}); +// end-example:03-contract + +// example:03-factory +const todoFactory: Factory = ({ list }, _principal, self) => { + const state = self.state({ rows: cell(Rows, []) }); + const derived = self.derive({ + openCount: derivation([state.rows], (rows) => rows.filter((todo) => !todo.done).length), + }); + return { + label: () => `Todos for ${list}`, + rows: state.rows, + openCount: derived.openCount, + add: ({ text }, ctx) => { + state.rows.change(ctx, (draft) => { + draft.push({ text, done: false }); + }); + return null; + }, + finish: ({ index }, ctx) => { + state.rows.change(ctx, (draft) => { + const todo = draft[index]; + if (!todo) throw invalid("args", "No such todo"); + todo.done = true; + }); + return null; + }, + reset: (rows, ctx) => { + state.rows.replace(ctx, rows, { diff: true }); + return null; + }, + }; +}; +// end-example:03-factory + +// example:05-views +const Todos = remote("tutorial.todo-view", { + version: 1, + input: Type.Object({ list: Type.String() }), + variants: { + viewer: { state: { rows: Rows } }, + editor: { + state: { rows: Rows }, + methods: { add: method(Type.Object({ text: Type.String() }), Type.Null()) }, + }, + }, +}); + +function serveViews(api: Pick): void { + api.serve(Todos, ({ list }, principal, self) => { + if (list !== "family") throw forbidden("This list is not shared"); + const store = self.get(TodoStore, { list }); + if (principal !== "owner" && principal !== hostPrincipal) { + return { kind: "viewer", rows: store.rows }; + } + return { + kind: "editor", + rows: store.rows, + add: ({ text }, ctx) => { + if (text.trim().length === 0) throw invalid("args", "A todo needs text"); + return store.add({ text }, ctx); + }, + }; + }); +} +// end-example:05-views + +it("01: objects live inside hosts", { timeout: 5000 }, async () => { + checkExamples("01"); + // example:01 + const host = createHost({ name: "todo-server" }); + assert.equal(host.name, "todo-server"); + await host.close(); + // end-example:01 +}); + +it("02: first contract, factory, handle and root", { timeout: 5000 }, async () => { + checkExamples("02"); + onTestFinished(() => host.close()); + // example:02-call + const host = createHost(); + host.serve(TodoLabel, ({ list }) => ({ label: () => `Todos for ${list}` })); + const result = await host.root(async (ctx) => { + const todos = ctx.get(TodoLabel, { list: "family" }); + return todos.label(null, ctx); + }); + assert.equal(result, "Todos for family"); + // end-example:02-call +}); + +it("03: cells publish values and derivations", { timeout: 5000 }, async () => { + checkExamples("03"); + onTestFinished(() => host.close()); + // example:03-call + const host = createHost({ dev: true }); + host.serve(TodoStore, todoFactory); + const todos = host.get(TodoStore, { list: "family" }); + await todos.ready; + const counts: (number | undefined)[] = []; + const stop = todos.openCount.subscribe((value) => counts.push(value)); + await host.root(async (ctx) => { + await todos.add({ text: "Buy milk" }, ctx); + await todos.finish({ index: 0 }, ctx); + await todos.reset([], ctx); + }); + assert.deepEqual(counts, [0, 1, 0]); + assert.deepEqual(todos.rows.value, []); + stop(); + todos.release(); + // end-example:03-call +}); + +it("04: holds, asynchronous resources and release", { timeout: 5000 }, async () => { + checkExamples("04"); + onTestFinished(() => host.close()); + // example:04-database + const Database = local("tutorial.database", { + version: 1, + input: Type.Object({}), + members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, + }); + const SavedTodos = local("tutorial.saved-todos", { + version: 1, + input: Type.Object({ list: Type.String() }), + members: { add: method(Type.Object({ text: Type.String() }), Type.Null()) }, + }); + const host = createHost(); + let opens = 0; + const closed = Promise.withResolvers(); + host.serve(Database, (_input, _principal, self) => { + const res = self.resources({ + db: resource({ + rev: 1, + open: async () => { + opens++; + return { saved: [] as string[] }; + }, + close: () => closed.resolve(), + }), + }); + return { db: self.expose(res.db) }; + }); + // end-example:04-database + // example:04-saving + host.serve(TodoStore, todoFactory); + host.serve( + SavedTodos, + ({ list }, _principal, self) => { + const database = self.get(Database, {}); + const todos = self.get(TodoStore, { list }); + return { + add: async ({ text }, ctx) => { + database.db!.saved.push(text); + await todos.add({ text }, ctx); + return null; + }, + }; + }, + { linger: 0 }, + ); + // end-example:04-saving + // example:04-call + const saved = host.get(SavedTodos, { list: "family" }); + const database = host.get(Database, {}); + await saved.ready; + await host.root((ctx) => saved.add({ text: "Buy milk" }, ctx)); + assert.deepEqual(database.db!.saved, ["Buy milk"]); + assert.equal(opens, 1); + const statuses: string[] = []; + saved.onStatus((status) => statuses.push(status)); + database.release(); + saved.release(); + await closed.promise; + assert.equal(saved.status, "released"); + assert.ok(statuses.includes("released")); + // end-example:04-call +}); + +it("05: principals select views of one shared store", { timeout: 5000 }, async () => { + checkExamples("05"); + onTestFinished(() => host.close()); + // example:05-call + const host = createHost({ principalKey: (principal) => String(principal) }); + host.serve(TodoStore, todoFactory); + serveViews(host); + const owner = host.get(Todos, { list: "family" }, { principal: "owner" }); + const guest = host.get(Todos, { list: "family" }, { principal: "guest" }); + await Promise.all([owner.ready, guest.ready]); + assert.equal(owner.kind, "editor"); + assert.equal(guest.kind, "viewer"); + await host.root((ctx) => owner.add({ text: "Buy milk" }, ctx), { principal: "owner" }); + assert.deepEqual(guest.rows.value, [{ text: "Buy milk", done: false }]); + await assert.rejects(guest.add({ text: "Not allowed" }), (error) => isChordError(error, "not_in_variant")); + await assert.rejects(owner.add({ text: " " }), (error) => isChordError(error, "invalid")); + // end-example:05-call +}); + +it("06: remote methods and live state over a memory link", { timeout: 5000 }, async () => { + checkExamples("06"); + onTestFinished(() => server.close()); + onTestFinished(() => browser.close()); + // example:06 + const server = createHost({ name: "server" }); + server.serve(TodoStore, todoFactory); + serveViews(server); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: "owner", kind: "client" }); + const browser = createHost({ name: "browser", kind: "client", upstream: up }); + const todos = browser.get(Todos, { list: "family" }, { at: "server" }); + const hydrated = Promise.withResolvers(); + const updated = Promise.withResolvers(); + const stop = todos.rows.subscribe((rows) => { + if (rows !== undefined) hydrated.resolve(); + if (rows?.length === 1) updated.resolve(); + }); + await todos.ready; + await hydrated.promise; + assert.deepEqual(todos.rows.value, []); + await browser.root((ctx) => todos.add({ text: "Buy milk" }, ctx)); + await updated.promise; + assert.deepEqual(todos.rows.value, [{ text: "Buy milk", done: false }]); + // Deliberately bypass TypeScript to simulate a malformed caller. + await assert.rejects(todos.add({ text: 123 } as never), (error) => isChordError(error, "invalid")); + stop(); + assert.equal(todos.rows.value, undefined); + todos.release(); + // end-example:06 +}); + +it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async () => { + checkExamples("07"); + onTestFinished(() => host.close()); + // example:07 + const Activity = local("tutorial.activity", { + version: 1, + input: Type.Object({}), + members: { seen: Type.Number() }, + }); + const host = createHost(); + host.serve(TodoStore, todoFactory); + const observed = Promise.withResolvers(); + const seeded = Promise.withResolvers(); + const stopped = Promise.withResolvers(); + host.serve(Activity, (_input, _principal, self) => { + const todos = self.get(TodoStore, { list: "family" }); + const state = self.state({ seen: cell(Type.Number(), 0) }); + self.listen({ + rows: listener(todos.rows, (rows, ctx) => { + state.seen.replace(ctx, rows?.length ?? 0); + if (rows?.length === 1) observed.resolve(); + }), + }); + self.tasks({ + seed: async (ctx) => { + await todos.add({ text: "Review the list" }, ctx); + seeded.resolve(); + await new Promise((resolve) => { + if (ctx.signal.aborted) resolve(); + else ctx.signal.addEventListener("abort", () => resolve(), { once: true }); + }); + stopped.resolve(); + }, + }); + return { seen: state.seen }; + }); + const activity = host.get(Activity, {}); + await activity.ready; + await Promise.all([seeded.promise, observed.promise]); + assert.equal(activity.seen.value, 1); + activity.release(); + await stopped.promise; + // end-example:07 +}); + +it("08: plugins contribute registry entries and expose callbacks", { timeout: 5000 }, async () => { + checkExamples("08"); + onTestFinished(() => host.close()); + // example:08 + const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); + const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); + let click: (() => Promise) | undefined; + const storeFacet: Facet = { + name: "todo-store", + ...defineFacet((api) => api.serve(TodoStore, todoFactory)), + }; + const menuFacet: Facet = { + name: "todo-menu", + ...defineFacet( + (api) => { + api.serve( + Menu, + (_input, _principal, self) => { + const todos = self.get(TodoStore, { list: "family" }); + self.register(Actions, { + addSample: async (ctx) => { + await todos.add({ text: "Try plugins" }, ctx); + return todos.openCount.value!; + }, + }); + return {}; + }, + { eager: true }, + ); + click = () => api.root((ctx) => api.registry(Actions).get("addSample", ctx)!(ctx)); + }, + { uses: [TodoStore] }, + ), + }; + const host = createHost(); + await host.load([menuFacet, storeFacet], { awaitEager: false }); + await host.ready(); + assert.equal(await click!(), 1); + assert.equal(host.registry(Actions).table().size, 1); + // end-example:08 +}); + +it("09: compatible reload, migration and an old-contract adapter", { timeout: 5000 }, async () => { + checkExamples("09"); + onTestFinished(() => host.close()); + // example:09-compatible + const host = createHost(); + const facet = (label: string): Facet => ({ + name: "todos", + setup: (api) => + api.serve(TodoStore, (input, principal, self) => ({ + ...todoFactory(input, principal, self), + label: () => label, + })), + }); + await host.load([facet("Original")]); + const todos = host.get(TodoStore, { list: "family" }); + await todos.add({ text: "Buy milk" }); + const entered = Promise.withResolvers(); + const resume = Promise.withResolvers(); + const oldRoot = host.root(async (ctx) => { + const first = await todos.label(null, ctx); + entered.resolve(); + await resume.promise; // Test-only pause: pin an old caller during reload. + return [first, await todos.label(null, ctx)]; + }); + await entered.promise; + const compatible = await host.reload([facet("Updated")]); + assert.deepEqual(compatible.gated, []); + assert.equal(await todos.label(null), "Updated"); + resume.resolve(); + assert.deepEqual(await oldRoot, ["Original", "Original"]); + assert.equal(todos.rows.value?.[0]?.text, "Buy milk"); + // end-example:09-compatible + // example:09-migrate + const events: string[] = []; + const stopEvents = todos.rows.events((event) => events.push(event.type)); + const migrated = await host.reload([ + { + name: "todos", + setup: (api) => + api.serve(TodoStore, todoFactory, { + state: { + version: 2, + migrate: { + rows: (old, from) => { + assert.equal(from, 1); + return (old as Static).map((todo) => ({ ...todo, text: todo.text.toUpperCase() })); + }, + }, + }, + }), + }, + ]); + assert.equal(migrated.gated.length, 1); + assert.deepEqual(todos.rows.value, [{ text: "BUY MILK", done: false }]); + assert.deepEqual(events, ["snap", "reset"]); + stopEvents(); + // end-example:09-migrate + // example:09-contract + // Remove label from the new provider's contract: it is now called heading. + const { label: _label, ...members } = TodoStore.def.members; + const RenamedStore = local("tutorial.todos", { + version: 2, + input: TodoStore.def.input, + members: { ...members, heading: method(Type.Null(), Type.String()) }, + }); + await host.reload([ + { + name: "todos", + setup: (api) => + api.serve( + RenamedStore, + (input, principal, self) => { + const { label, ...behavior } = todoFactory(input, principal, self); + return { ...behavior, heading: label }; + }, + { + state: { version: 2 }, + accepts: { + 1: (current) => ({ ...current, label: current.heading }), + }, + }, + ), + }, + ]); + assert.equal(await todos.label(null), "Todos for family"); + const newer = host.get(RenamedStore, { list: "family" }); + assert.equal(await newer.heading(null), "Todos for family"); + assert.equal(todos.rows.value?.[0]?.text, "BUY MILK"); + // end-example:09-contract +}); + +it("10: stub a provider with a fake facet and check error codes", { timeout: 5000 }, async () => { + checkExamples("10"); + onTestFinished(() => host.close()); + // example:10 + const host = createHost({ dev: true }); + const received: string[] = []; + const fake: Facet = { + name: "fake-store", + ...defineFacet((api: FacetApi) => { + api.serve(TodoStore, (input, principal, self) => ({ + ...todoFactory(input, principal, self), + add: ({ text }) => { + received.push(text); + return null; + }, + })); + }), + }; + await host.load([fake]); + serveViews(host); + const owner = host.get(Todos, { list: "family" }, { principal: "owner" }); + await owner.add({ text: "Test delegation" }); + assert.deepEqual(received, ["Test delegation"]); + await assert.rejects(owner.add({ text: "" }), (error) => isChordError(error, "invalid")); + owner.release(); + await assert.rejects(owner.add({ text: "Too late" }), (error) => isChordError(error, "released")); + // end-example:10 +}); + +it("11: short roots release temporary holds", { timeout: 5000 }, async () => { + checkExamples("11"); + onTestFinished(() => host.close()); + // example:11 + const host = createHost(); + host.serve(TodoStore, todoFactory); + const screen = host.get(TodoStore, { list: "family" }); + await screen.ready; + await host.root(async (ctx) => { + const temporary = ctx.get(TodoStore, { list: "family" }); + await temporary.add({ text: "Keep roots short" }, ctx); + }); // The temporary handle is released here; screen still holds the store. + assert.equal(screen.openCount.value, 1); + screen.release(); // No hold remains; without persistence, disposal loses these rows. + // end-example:11 +}); diff --git a/packages/chord/test/websocket.test.ts b/packages/chord/test/websocket.test.ts new file mode 100644 index 000000000..fae2f5d7c --- /dev/null +++ b/packages/chord/test/websocket.test.ts @@ -0,0 +1,101 @@ +import assert from "node:assert/strict"; +import type { IncomingMessage } from "node:http"; +import { Type } from "typebox"; +import { describe, it } from "vitest"; +import { WebSocketServer } from "ws"; +import { + acceptWebSockets, + cell, + createHost, + isChordError, + method, + remote, + type WebSocketLike, + webSocketLink, +} from "../src/index.ts"; + +const Board = remote("t.board", { + version: 1, + input: Type.Object({}), + variants: { + main: { + state: { notes: Type.Array(Type.String()) }, + methods: { add: method(Type.String(), Type.Null()), who: method(Type.Null(), Type.Unknown()) }, + }, + }, +}); + +describe("WebSocket links", () => { + it("serves a browser-style client over a real socket, stamped by the authenticator", async () => { + const server = createHost({ name: "server", onError: () => {} }); + server.serve(Board, (_input, principal, self) => { + const state = self.state({ notes: cell(Type.Array(Type.String()), []) }); + return { + kind: "main", + notes: state.notes, + add: (text, ctx) => { + state.notes.change(ctx, (draft) => { + draft.push(text); + }); + return null; + }, + who: () => principal, + }; + }); + const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); + await new Promise((resolve) => wss.once("listening", () => resolve())); + acceptWebSockets(wss as never, server, { + principal: (request: IncomingMessage) => { + const token = new URL(request.url ?? "/", "http://x").searchParams.get("token"); + return token === "ok" ? { guest: "web" } : undefined; + }, + }); + const port = (wss.address() as { port: number }).port; + const open = (token: string) => + createHost({ + name: "browser", + kind: "client", + upstream: webSocketLink( + new WebSocket(`ws://127.0.0.1:${port}/?token=${token}`) as unknown as WebSocketLike, + ), + onError: () => {}, + }); + try { + const browser = open("ok"); + const board = browser.get(Board, {}, { at: "server" }); + const seen: unknown[] = []; + board.notes.subscribe((value) => seen.push(value)); + await board.add("hello"); + assert.deepEqual(await board.who(null), { guest: "web" }); + for (let i = 0; i < 50 && board.notes.value?.length !== 1; i++) await new Promise((r) => setTimeout(r, 5)); + assert.deepEqual(board.notes.value, ["hello"]); + assert.deepEqual(seen.at(-1), ["hello"]); + await browser.close(); + + // WebSocket peers are always clients: one announcing itself as a host is cut off. + const impostor = createHost({ + name: "session:S1", + upstream: webSocketLink(new WebSocket(`ws://127.0.0.1:${port}/?token=ok`) as unknown as WebSocketLike), + onError: () => {}, + }); + await assert.rejects( + impostor.root((ctx) => impostor.get(Board, {}, { at: "server" }).add("x", ctx), { + signal: AbortSignal.timeout(150), + }), + (error) => isChordError(error, "aborted"), + ); + await impostor.close(); + + const refused = open("bad"); + const denied = refused.get(Board, {}, { at: "server" }); + // The server closes unauthenticated sockets; calls wait for a reconnect until their ctx gives up. + await assert.rejects( + refused.root((ctx) => denied.add("x", ctx), { signal: AbortSignal.timeout(100) }), + (error) => isChordError(error, "aborted"), + ); + await refused.close(); + } finally { + await new Promise((resolve) => wss.close(() => resolve())); + } + }); +}); diff --git a/packages/durable/docs/chord-delta-findings.md b/packages/durable/docs/chord-delta-findings.md index 3f6a88558..cb9f596f0 100644 --- a/packages/durable/docs/chord-delta-findings.md +++ b/packages/durable/docs/chord-delta-findings.md @@ -323,7 +323,7 @@ implementation or reproduces the historical graph/mutation comparison. Targeted tests from `packages/chord`: ```sh -node ../../node_modules/vitest/dist/cli.js --run test/delta.test.ts test/delta-clone.test.ts test/delta-tracker/retention.test.ts test/services.test.ts test/service-wire.test.ts test/facets.test.ts +node ../../node_modules/vitest/dist/cli.js --run test/delta.test.ts test/delta-clone.test.ts test/delta-tracker/retention.test.ts test/replicated-state.test.ts ``` Historical source identifiers: diff --git a/packages/durable/test/session-states.test.ts b/packages/durable/test/session-states.test.ts index 4c3678b46..40edcd3ce 100644 --- a/packages/durable/test/session-states.test.ts +++ b/packages/durable/test/session-states.test.ts @@ -1,6 +1,5 @@ import { defineDoc, defineDocFamily } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; -import { getReplicatedStateInternals } from "../../chord/src/services/state-internals.ts"; import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; type State = { value: number; retained: { label: string } }; @@ -81,20 +80,15 @@ describe("Session document states", () => { second.dispose(); }); - it("shares exact committed value and operation references with Chord", async () => { + it("shares exact committed value references with Chord", async () => { const { session, publications } = await createState(); const state = (await session.documentState(StateDoc, context))!; - let receivedOps: unknown; - getReplicatedStateInternals(state)!.subscribe((ops) => { - receivedOps = ops; - }); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 4; }, context); await flush(); const published = documentChanges(publications.at(-1)!)[0]!; expect(state.value).toBe(published.value); - expect(receivedOps).toBe(published.ops); expect(state.value?.retained).toBe((await session.snapshot(StateDoc, context))!.retained); state.dispose(); }); @@ -116,17 +110,12 @@ describe("Session document states", () => { it("publishes null retirement and never follows a replacement incarnation", async () => { const { session } = await createState(); const oldState = (await session.documentState(StateDoc, context))!; - let retirementOps: unknown; - getReplicatedStateInternals(oldState)!.subscribe((ops) => { - retirementOps = ops; - }); await session.commit(async (tx) => { await tx.retireDoc(StateDoc); (await tx.doc(StateDoc)).value = 10; }, context); await flush(); expect(oldState.value).toBeNull(); - expect(retirementOps).toEqual([["r", null]]); const replacement = (await session.documentState(StateDoc, context))!; expect(replacement.value?.value).toBe(10); diff --git a/scripts/browser-smoke-entry.ts b/scripts/browser-smoke-entry.ts index 84f257788..4c6d07c38 100644 --- a/scripts/browser-smoke-entry.ts +++ b/scripts/browser-smoke-entry.ts @@ -1,3 +1,4 @@ +import { createHost, memoryLinkPair, replicatedState } from "@earendil-works/chord"; import { createAssistantMessageEventStream, Type } from "@earendil-works/pi-ai"; import { complete, getModel, getProviders, streamSimple } from "@earendil-works/pi-ai/compat"; import { Agent, streamProxy } from "@earendil-works/pi-agent-core"; @@ -19,4 +20,7 @@ console.log( typeof stream.push, agent.hasQueuedMessages(), typeof streamProxy, + typeof createHost, + typeof memoryLinkPair, + replicatedState({ browser: true }).value.browser, ); diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index 17b48c408..da4cce2ee 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -58,8 +58,9 @@ const BUDGETS = { "packages/durable": { ".": { // Tool argument validation reaches TypeBox; provider-session creation reaches pi-ai's lean UUID utility; the - // memory and SQLite storages share scan order and cursor handling. - maxFiles: 63, + // memory and SQLite storages share scan order and cursor handling. Values from Chord's root (copyJson, + // replicatedState) reach its host runtime through the root barrel. + maxFiles: 66, forbid: [ "packages/ai/src/index.ts", "packages/ai/src/utils/typebox-helpers.ts", diff --git a/tsconfig.json b/tsconfig.json index b0868c4e4..14f1d161d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,7 +6,6 @@ "moduleResolution": "NodeNext", "paths": { "@earendil-works/chord": ["./packages/chord/src/index.ts"], - "@earendil-works/chord/bundler": ["./packages/chord/src/bundler.ts"], "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], diff --git a/vitest.base.ts b/vitest.base.ts index 4eaff52e1..d450260c2 100644 --- a/vitest.base.ts +++ b/vitest.base.ts @@ -5,7 +5,6 @@ export const workspaceSourcePaths = { chordIndex: fileURLToPath(new URL("./packages/chord/src/index.ts", import.meta.url)), chordContext: fileURLToPath(new URL("./packages/chord/src/context/index.ts", import.meta.url)), chordDelta: fileURLToPath(new URL("./packages/chord/src/delta/index.ts", import.meta.url)), - chordBundler: fileURLToPath(new URL("./packages/chord/src/bundler.ts", import.meta.url)), chordNode: fileURLToPath(new URL("./packages/chord/src/node.ts", import.meta.url)), telemetryIndex: fileURLToPath(new URL("./packages/telemetry/src/index.ts", import.meta.url)), telemetryTesting: fileURLToPath(new URL("./packages/telemetry/src/testing/index.ts", import.meta.url)), @@ -32,7 +31,6 @@ export default defineConfig({ { find: /^@earendil-works\/chord$/, replacement: workspaceSourcePaths.chordIndex }, { find: /^@earendil-works\/chord\/context$/, replacement: workspaceSourcePaths.chordContext }, { find: /^@earendil-works\/chord\/delta$/, replacement: workspaceSourcePaths.chordDelta }, - { find: /^@earendil-works\/chord\/bundler$/, replacement: workspaceSourcePaths.chordBundler }, { find: /^@earendil-works\/chord\/node$/, replacement: workspaceSourcePaths.chordNode }, { find: /^@earendil-works\/pi-telemetry$/, replacement: workspaceSourcePaths.telemetryIndex }, { find: /^@earendil-works\/pi-telemetry\/testing$/, replacement: workspaceSourcePaths.telemetryTesting },