From ff33ef42c4bd762cecd6958e3aabbc8e684ceb05 Mon Sep 17 00:00:00 2001
From: Mario Zechner
Date: Wed, 4 Nov 2026 11:59:14 +0000
Subject: [PATCH] feat(chord): telemetry from pages and workers stored by the
server, who a span was for
The four telemetry gaps (rows on commits, chord.host, microsecond times, a read-only store) and: pages record and
send span batches over their link to the server's ingest object (checked, capped, rate-limited, stamped browser and
with the link's principal); workers send theirs too, so the server is the store's only writer; principalAttributes
lets an app choose what a span records about its principal. The tutorial shows a real page's click traced to the
worker, with the trace viewer.
---
packages/chord/CHANGELOG.md | 5 +-
packages/chord/docs/design.md | 53 ++-
.../chord/docs/guide-plugins-hosts/book.md | 43 +-
.../chord/docs/guide-plugins-hosts/index.html | 44 +-
packages/chord/docs/tutorial/book.md | 422 +++++++++++-------
packages/chord/docs/tutorial/index.html | 399 ++++++++++-------
.../docs/tutorial/screens/click-after.png | Bin 0 -> 18172 bytes
.../docs/tutorial/screens/click-before.png | Bin 0 -> 16248 bytes
.../docs/tutorial/screens/traces-ada.png | Bin 0 -> 38968 bytes
.../chord/docs/tutorial/screens/traces.png | Bin 81083 -> 92380 bytes
packages/chord/package.json | 5 +
packages/chord/src/documents/documents.ts | 4 +-
packages/chord/src/documents/transaction.ts | 16 +-
packages/chord/src/host/attempt.ts | 16 +-
packages/chord/src/host/ctx.ts | 31 +-
packages/chord/src/host/facet.ts | 5 +
packages/chord/src/host/host.ts | 94 +++-
packages/chord/src/host/index.ts | 1 +
packages/chord/src/host/native.ts | 11 +-
packages/chord/src/host/object.ts | 2 +
packages/chord/src/host/root.ts | 7 +
packages/chord/src/host/route.ts | 6 +-
packages/chord/src/host/routines.ts | 1 +
packages/chord/src/host/staging.ts | 2 +-
packages/chord/src/jobs/job.ts | 15 +-
packages/chord/src/node/app/app.ts | 48 +-
packages/chord/src/node/app/worker.ts | 45 +-
packages/chord/src/node/http/http.ts | 11 +-
packages/chord/src/remote/relay.ts | 14 +-
packages/chord/src/remote/remote-object.ts | 4 +
packages/chord/src/sqlite/node.ts | 19 +-
packages/chord/src/store/step.ts | 6 +-
packages/chord/src/store/store.ts | 11 +-
packages/chord/src/store/table-writes.ts | 7 +-
packages/chord/src/surfaces/client.ts | 12 +-
packages/chord/src/telemetry-ingest/batch.ts | 136 ++++++
packages/chord/src/telemetry-ingest/facet.ts | 124 +++++
packages/chord/src/telemetry-ingest/index.ts | 20 +
.../chord/src/telemetry-ingest/recorder.ts | 348 +++++++++++++++
packages/chord/src/telemetry-ingest/sender.ts | 101 +++++
packages/chord/src/telemetry-sqlite/index.ts | 419 +++++------------
packages/chord/src/telemetry-sqlite/node.ts | 34 ++
packages/chord/src/telemetry-sqlite/schema.ts | 69 +++
packages/chord/src/telemetry-sqlite/store.ts | 36 +-
packages/chord/src/telemetry.ts | 43 +-
packages/chord/src/web/README.md | 4 +
packages/chord/src/web/page.ts | 34 +-
.../chord/test/documents/documents.test.ts | 43 +-
.../chord/test/examples/61-tutorial/main.ts | 46 +-
.../examples/61-tutorial/plugins/lists/web.ts | 10 +-
.../61-tutorial/plugins/traces/clock.test.ts | 60 +++
.../61-tutorial/plugins/traces/clock.ts | 42 ++
.../61-tutorial/plugins/traces/server.ts | 67 +--
.../test/examples/61-tutorial/screenshots.ts | 96 ++--
.../examples/61-tutorial/src/api/v1/server.ts | 7 +-
.../examples/61-tutorial/src/api/v1/web.ts | 2 +-
.../examples/61-tutorial/src/principals.ts | 20 +
.../examples/61-tutorial/src/telemetry.ts | 6 +-
.../test/examples/61-tutorial/src/web/main.ts | 2 +
.../examples/61-tutorial/src/web/style.css | 3 +-
.../examples/61-tutorial/src/web/traces.ts | 42 +-
.../chord/test/examples/61-tutorial/start.ts | 14 +-
.../chord/test/examples/61-tutorial/trace.ts | 33 +-
.../chord/test/examples/61-tutorial/worker.ts | 9 +-
.../chord/test/examples/62-telemetry/main.ts | 22 +-
.../chord/test/examples/62-telemetry/start.ts | 8 +-
.../test/examples/62-telemetry/worker.ts | 11 +-
packages/chord/test/host/ctx.test.ts | 2 +-
packages/chord/test/host/facet.test.ts | 40 +-
packages/chord/test/host/host.test.ts | 60 ++-
packages/chord/test/host/native.test.ts | 5 +-
packages/chord/test/host/root.test.ts | 41 ++
packages/chord/test/host/route.test.ts | 7 +-
packages/chord/test/host/routines.test.ts | 9 +-
packages/chord/test/jobs/job.test.ts | 18 +-
packages/chord/test/jobs/persisted.test.ts | 13 +-
.../chord/test/node/fixtures/web/telemetry.ts | 53 +++
packages/chord/test/node/http/http.test.ts | 4 +-
packages/chord/test/node/web/page.test.ts | 50 ++-
packages/chord/test/node/web/support.ts | 5 +-
packages/chord/test/remote/provider.test.ts | 2 +
.../chord/test/remote/remote-object.test.ts | 1 +
packages/chord/test/sqlite/node.test.ts | 34 ++
.../chord/test/telemetry-ingest/facet.test.ts | 189 ++++++++
.../test/telemetry-ingest/recorder.test.ts | 113 +++++
.../test/telemetry-ingest/sender.test.ts | 98 ++++
.../chord/test/telemetry-sqlite-node.test.ts | 72 +++
packages/chord/test/telemetry-sqlite.test.ts | 89 +++-
packages/chord/test/telemetry.test.ts | 28 +-
packages/telemetry/CHANGELOG.md | 4 +
packages/telemetry/src/index.ts | 8 +-
packages/telemetry/src/memory.ts | 37 +-
packages/telemetry/src/render.ts | 16 +-
packages/telemetry/test/render.test.ts | 9 +
packages/telemetry/test/store.test.ts | 48 +-
95 files changed, 3358 insertions(+), 967 deletions(-)
create mode 100644 packages/chord/docs/tutorial/screens/click-after.png
create mode 100644 packages/chord/docs/tutorial/screens/click-before.png
create mode 100644 packages/chord/docs/tutorial/screens/traces-ada.png
create mode 100644 packages/chord/src/telemetry-ingest/batch.ts
create mode 100644 packages/chord/src/telemetry-ingest/facet.ts
create mode 100644 packages/chord/src/telemetry-ingest/index.ts
create mode 100644 packages/chord/src/telemetry-ingest/recorder.ts
create mode 100644 packages/chord/src/telemetry-ingest/sender.ts
create mode 100644 packages/chord/src/telemetry-sqlite/node.ts
create mode 100644 packages/chord/src/telemetry-sqlite/schema.ts
create mode 100644 packages/chord/test/examples/61-tutorial/plugins/traces/clock.test.ts
create mode 100644 packages/chord/test/examples/61-tutorial/plugins/traces/clock.ts
create mode 100644 packages/chord/test/node/fixtures/web/telemetry.ts
create mode 100644 packages/chord/test/telemetry-ingest/facet.test.ts
create mode 100644 packages/chord/test/telemetry-ingest/recorder.test.ts
create mode 100644 packages/chord/test/telemetry-ingest/sender.test.ts
create mode 100644 packages/chord/test/telemetry-sqlite-node.test.ts
diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md
index e6b0922b9..d1b3d216e 100644
--- a/packages/chord/CHANGELOG.md
+++ b/packages/chord/CHANGELOG.md
@@ -6,6 +6,7 @@
- Chord v2 is the package's root: `@earendil-works/chord` exports v2 (was `@earendil-works/chord/next`). Chord v1 moved unchanged to `@earendil-works/chord/old` (was the root) and `@earendil-works/chord/old/node` (was `/node`); it stays until p2 runs on v2. The foundations v1's root re-exported have their own subpaths: `@earendil-works/chord/json` (`JsonValue`, `JsonRepresentation`, `copyJson`, `isJsonValue`) and `@earendil-works/chord/watch` (`Watch`, `NativeWatch` and the watch types), beside `/context` and `/delta`. Migration: imports of `@earendil-works/chord/next` drop `/next`; v1's imports of `@earendil-works/chord` become `@earendil-works/chord/old`, and `/node` becomes `/old/node`; imports of `Context`, `Draft`, `JsonValue`, `copyJson` or the watch types from the root move to `/context`, `/delta`, `/json` and `/watch`. In the repository, v2's `src/next` is now `src`, `src/next-node` is `src/node`, `src/next-web`, `src/next-react` and `src/next-tui` are `src/web`, `src/react` and `src/tui`, v1 is in `src/old`, and tests and docs moved likewise (`test/next` to `test`, `docs/next` to `docs`, v1's to `test/old` and `docs/old`).
- Chord v1 (`@earendil-works/chord/old`, `@earendil-works/chord/old/node`) removed, with its tests and docs (`src/old`, `test/old`, `docs/old`). Apps still on v1 pin `@earendil-works/chord@1.5.0`.
+- Chord v2's `startWorker` no longer takes `telemetry` (it was added unreleased in slice 9): with a telemetry store on the server, the kit's workers send their spans and metrics to it, and no worker opens the store's file. `startServer`'s `telemetry` is a store with `ingest` (`createSqliteTelemetry`'s). Migration: drop `telemetry` from `startWorker` and the store's path from the worker's environment.
### Added
@@ -21,7 +22,9 @@
- Chord v2 gains live queries: `self.query(table, q, limit)` in the factory of a table's owner is a watchable of the query's rows (at most `limit`, in its order), usable as a state member and a derive source. Chord runs it before the object goes live and again inside the commit step of every commit of the object that writes the table, publishing only the diff in that commit's batch; a commit writing no table with a live query runs nothing (`chord.query.runs` counts the runs). Only the object's own tables; the limit is required (`invalid/args` at declaration). A run that fails (a row the code cannot read) is a failed member, reported once as the new `HostError` `query_failed { object, table, error }`, and heals at the next good run. A live query is no native, so an object showing table rows live swaps on reload instead of restarting, as reading them in `self.open` made it. Each touching commit runs the query once (100 rows: about 60 µs, 80 µs watched): not for tables written hundreds of times a second.
- Chord v2 gains messages and languages in the app kit (design §23): `defineMessages(namespace, { en: {...}, de: {...} })` returns a typed `t(key, ...args)` (the first catalog types the others: a missing key, an unknown one or a wrong argument is a type error; values are strings or functions), `t.message(key, ...args)` builds a message as data (`MessageData`, `{ namespace, key, params }`, for state members typed with `MessageSchema`), `t.render(message)` renders one (a foreign namespace or an unknown key from a link shows the key), `t.translations` lists its catalogs, and `MessagesOf` types a translation. The `Translations` registry takes a list of `{ namespace, locale, messages, override? }` per contributor (one catalog per namespace and locale: the first by plugin name wins, reported; `override: true` wins); a missing key falls back down the locale's chain (`pt-BR`, `pt`) to the first catalog, reported once. `LocaleChoice` keeps one locale per principal on the server (the theme choice's store, now shared as one preference); a client follows it with `followActiveLanguage`, defaulting to `navigator.languages` on a page and the system locale in the terminal (`startTerminal`'s `locales`), matched against the catalogs, switched live. `useMessages(t)` (`/react`) and `TuiScope.messages(t)` (`/tui`) render in the host's language and re-render or redraw on a change; `startPage` sets `lang` and `dir` on the page (`followLanguage`, `/web`); `formatTime` and `formatNumber` use `Intl` in the language being rendered. The kit serves `Translations` on the server, workers, pages and terminals and `LocaleChoice` on the server; client errors gain `{ kind: "messages", report }`.
-- Chord v2 gains telemetry across hosts (slice 9): `CHORD_TELEMETRY_SCHEMA` (root export) types every span, event and metric Chord records. With a host adapter (`createHost({ telemetry })`), calls (`chord.call`), methods (`chord.method`, local and remote), relays (`chord.relay`), holds (`chord.hold`), activations (`chord.activate`, ending `live`, `failed`, `parked` or `ended`, with `chord.retry` and `chord.park` events), natives' opens and closes, `load`, `reload` (a `chord.reload.fate` event per object), resolves (a worker start included), revocations, routine calls, job phases (`intent`, `run`, `finish`: a persisted run resumed in a new attempt or process stays under its stored carrier) and HTTP requests (`chord.http`: method, route pattern, status, never the path) are spans; failures carry a fixed category and the message cut at 500 characters, a cancel is no error; commits are `chord.commit { documents, ops, duration }` events of the span they ran under. `hold` and `call` carry the caller's trace carrier (`trace`, at most 8 keys, keys and values of at most 256 characters of printable ASCII, else the message fails the message schema and the link closes; the binary codec's kind byte flag `0x40`), relays forward their own span's, and the provider runs the method under it (an absent or unknown carrier: the adapter's root); a reference's carrier rides on its first hold only, a re-hold after a reconnect being a fresh root linked to it. A hold's `type` and its address's `kind` are bounded at 256 characters by the message schema, and span attributes a peer chose at 256 characters. `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child ctx. Host-wide metrics: `chord.link.bytes`, `chord.link.state_messages`, `chord.commit.duration`, `chord.calls.in_flight`, `chord.holds.active`, `chord.links.open`, `chord.park.duration`, `chord.backpressure.wait`. Without an adapter nothing is started, built or sent: a call over a link costs what it did before. `createHttp`, `startServer` and `startWorker` take a `telemetry` adapter (`createHttp` continues a W3C `traceparent` only from a request a contributed relay verified: an app's proxy must strip it from browsers). A host with an adapter counts the cost counters of its links' and root scopes' work on it also when the app passed a context without a span. The SQLite telemetry store keeps spans of one millisecond in the order they started.
+- Chord v2 gains telemetry across hosts (slice 9): `CHORD_TELEMETRY_SCHEMA` (root export) types every span, event and metric Chord records. With a host adapter (`createHost({ telemetry })`), calls (`chord.call`), methods (`chord.method`, local and remote), relays (`chord.relay`), holds (`chord.hold`), activations (`chord.activate`, ending `live`, `failed`, `parked` or `ended`, with `chord.retry` and `chord.park` events), natives' opens and closes, `load`, `reload` (a `chord.reload.fate` event per object), resolves (a worker start included), revocations, routine calls, job phases (`intent`, `run`, `finish`: a persisted run resumed in a new attempt or process stays under its stored carrier) and HTTP requests (`chord.http`: method, route pattern, status, never the path) are spans; failures carry a fixed category and the message cut at 500 characters, a cancel is no error; commits are `chord.commit { documents, ops, rows, duration }` events of the span they ran under. `hold` and `call` carry the caller's trace carrier (`trace`, at most 8 keys, keys and values of at most 256 characters of printable ASCII, else the message fails the message schema and the link closes; the binary codec's kind byte flag `0x40`), relays forward their own span's, and the provider runs the method under it (an absent or unknown carrier: the adapter's root); a reference's carrier rides on its first hold only, a re-hold after a reconnect being a fresh root linked to it. A hold's `type` and its address's `kind` are bounded at 256 characters by the message schema, and span attributes a peer chose at 256 characters. `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child ctx. Host-wide metrics: `chord.link.bytes`, `chord.link.state_messages`, `chord.commit.duration`, `chord.calls.in_flight`, `chord.holds.active`, `chord.links.open`, `chord.park.duration`, `chord.backpressure.wait`. Without an adapter nothing is started, built or sent: a call over a link costs what it did before. `createHttp`, `startServer` and `startWorker` take a `telemetry` adapter (`createHttp` continues a W3C `traceparent` only from a request a contributed relay verified: an app's proxy must strip it from browsers). A host with an adapter counts the cost counters of its links' and root scopes' work on it also when the app passed a context without a span. The SQLite telemetry store keeps spans of one microsecond in the order they started.
+- Chord v2's telemetry closes four gaps found writing the tutorial: `chord.commit` counts the table rows a commit's writes changed (`rows`; a commit writing only rows no longer reads `documents=0 ops=0`); every span Chord starts, `ctx.trace`'s too, carries `chord.host`, the host's address key or `client` without an address (`Host` gains `address`); the SQLite telemetry store keeps span times in whole microseconds from pi-telemetry's `telemetryNow` (store version 2: a recorder migrates a version 1 file in place, its rows kept); and `openTelemetryStore(path)` (new subpath `@earendil-works/chord/telemetry-sqlite/node`) reads a store file read-only, implementing `TelemetryStore` only, refusing a file without a store or with one of another version. `openNodeSqliteDatabase` takes `mode: "read-only"`.
+- Chord v2's telemetry is stored in one place, the server (decisions "Telemetry from browser pages", "The server is the one place telemetry is stored", "Who a span was for"). Pages and workers record in memory with Chord's `TelemetryRecorder` (the SQLite store's own recorder too) and send finished spans in batches over their link to the server's `PageTelemetry` and `WorkerTelemetry` (`ingest(batch)`), which the app kit serves when `startServer` has a telemetry store; the store gains `ingest` (never replacing a stored span). `startPage({ telemetry: true })` records the page and sends its spans every 3 s and when hidden; the server checks a page's batch strictly (at most 100 spans, every field bounded, ids printable ASCII, no control characters), keeps only spans under the page's own ids (`browser::`), caps them at 1,200 per page and minute, marks them `chord.host: browser` and stamps them with the principal of their link. A worker sends its spans and metrics every second, stamped with its host. `host.root(fn, ctx, { trace: { name, attributes } })` runs `fn` in a new root span (a click). Method and call spans record whom they were for: `chord.principal` (the principal key) by default, or what the host's new `principalAttributes` hook returns (`createHost`, `startServer`, `startWorker`). `ServeOptions.untraced` serves a type whose calls start no method span. `Host` gains `incarnation` and `principalAttributes`.
### Changed
diff --git a/packages/chord/docs/design.md b/packages/chord/docs/design.md
index f3c165e54..a28987f01 100644
--- a/packages/chord/docs/design.md
+++ b/packages/chord/docs/design.md
@@ -2769,7 +2769,9 @@ Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`;
`chord.open` (under the activation), `chord.close` (a root), `chord.load`, `chord.reload` (a `chord.reload.fate`
event per object), `chord.resolve`, `chord.revoke` (recorded after the synchronous revocation, its `count`),
`chord.routine` (a root per call), `chord.job` (`intent` under the starter, `run` and `finish` under the start's
- carrier), `chord.http`. Outcomes are attributes; a failure is `status: "error"` with a fixed category (`call_failed`,
+ carrier), `chord.http`. Every span Chord starts, `ctx.trace`'s too, names the host that recorded it: `chord.host`, the
+ host's address key (`list|groceries`, `server`), `client` without an address, capped like the other attributes (a
+ viewer tells hosts apart by it, never by a store's span ids). Outcomes are attributes; a failure is `status: "error"` with a fixed category (`call_failed`,
`method_failed`, `relay_failed`, `hold_failed`, `activation_failed`, `open_failed`, `close_failed`, `load_failed`,
`reload_failed`, `resolve_failed`, `routine_failed`, `job_failed`, `http_failed` for a status of 500 or more,
`trace_failed`) and the message cut at 500 characters; a cancel (the caller's signal, `aborted`, an open its attempt
@@ -2796,8 +2798,10 @@ Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`;
starts no span, builds no attribute object and injects no carrier; with the NOOP adapter a call over a link is within
noise of the code before slice 9 (`benchmarks.md`). A recording host's call costs its spans (`OpenSpan`, a span kept
open until its work's promise settles).
-- **Events and metrics:** `chord.commit { documents, ops, duration }` on the span of the context a transaction ran
- under (a method, a routine, a job phase, an HTTP request), only for a commit that changed something; the host-wide
+- **Events and metrics:** `chord.commit { documents, ops, rows, duration }` on the span of the context a transaction
+ ran under (a method, a routine, a job phase, an HTTP request), only for a commit that changed something; `rows` is the
+ table rows its writes changed, as the commit step already counts them (a `put` one, an update or delete the rows its
+ statement changed; nothing is measured for it); the host-wide
metrics go to the host's adapter (`host.metrics`), not to a context: `chord.link.bytes { direction }` (every encoded
message of the host's links), `chord.link.state_messages`, `chord.commit.duration`, `chord.calls.in_flight`,
`chord.holds.active`, `chord.links.open`, `chord.park.duration`, `chord.backpressure.wait`. The cost counters still
@@ -2809,10 +2813,45 @@ Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`;
carrier `{ traceparent, tracestate }`) only from a request a contributed relay verified, each header at most 256 and
512 characters of printable ASCII (else ignored). A verified relay vouches for the proxy, not the browser behind it:
an app's proxy must strip `traceparent` and `tracestate` from browsers' requests (or set its own), or a browser
- chooses the trace its requests join. `startServer({ telemetry })` passes the adapter to
- its host and HTTP; `startWorker({ telemetry })` to its host. Each process exports to the store it is given; a shared
- SQLite file per data directory is the app's choice (example 62). The SQLite store orders spans of one millisecond as
- their recorder started them (by start time, then id length and id).
+ chooses the trace its requests join. `startServer({ telemetry })` passes the store to its host and HTTP and serves
+ the ingest objects; workers and recording pages send it their spans (below). The SQLite store orders spans of one
+ microsecond as their recorder started them (by start time, then id length and id).
+- **Span times (after slice 9):** a span's start is `telemetryNow()` (pi-telemetry): wall-clock milliseconds at
+ microsecond resolution from `performance.now()` anchored to the wall clock, anchored anew when the two part by more
+ than 10 ms (a clock set, a sleep the monotonic clock did not count); its end and its events' times are the monotonic
+ clock's from there. The in-memory adapter and the SQLite store share it, and `renderTrace` prints lengths to the
+ microsecond. The SQLite store keeps span times in whole microseconds (`started_us`, `ended_us`): store version 2, to
+ which a recorder migrates a version 1 file in place (its rows' milliseconds multiplied), since several processes
+ share one file and a new store version would strand the rows already there.
+- **The server is the one place telemetry is stored** (decisions "Telemetry from browser pages", "The server is the
+ one place telemetry is stored", "Who a span was for"; `src/telemetry-ingest/`): the server's store is the only
+ writer of its file. Pages and workers record with Chord's in-memory `TelemetryRecorder` (the SQLite store records
+ with it too: one carrier format, one bounded queue of settled spans and metric aggregates per 10 s) and send finished
+ spans in batches over the link they already have to the server's `PageTelemetry` and `WorkerTelemetry`
+ (`ingest(batch)`), served by the kit when `startServer` has a telemetry store (`TelemetryContext` with the store's
+ `ingest`). A worker sends every second (and its last spans and partial metrics when it closes), written as they are,
+ stamped with its host (`chord.host`: its address key, known from the principal its link stamps). A page records
+ when `startPage({ telemetry: true })` and sends spans only, every 3 s and when hidden; the server treats it as
+ untrusted: the batch schema bounds everything (100 spans; names 128 characters; 32 attributes, keys 64, values 256
+ characters or numbers or booleans; 16 events; 8 links; error messages 500; ids printable ASCII; no control
+ characters anywhere), a span counts only when its id, trace and parent are under the page's own prefix
+ (`browser::`, so it neither writes into another sender's trace nor poses as a server span),
+ 1,200 spans per page and minute (past it dropped, counted in the answer), each marked `chord.host: browser` and
+ stamped with the principal attributes of its link, overriding what the page wrote. The store writes ingested spans
+ with `INSERT OR IGNORE`: a stored span is never replaced. The ingest objects' calls start no `chord.method`
+ (`ServeOptions.untraced`): a span per batch would be telemetry about telemetry. A trace of a page's own starts with
+ `host.root(fn, ctx, { trace: { name, attributes } })`, a root span of the host's adapter (`trace_failed` on a throw).
+ A viewer clamps a page's spans against the server spans under them (the page's clock may be off).
+- **Who a span was for:** `chord.method` and `chord.call` record their principal's attributes: by default
+ `chord.principal`, its principal key (capped); a host's `principalAttributes(principal)` hook replaces them (none,
+ the key, more: a user's name the app resolves). Chord calls it for every span it records with a principal, so it is
+ synchronous and cheap, its caching its own; the kit passes `startServer`'s and `startWorker`'s to their hosts, and
+ the server stamps a page's spans with it for the link's principal. `Host` exposes `principalAttributes`,
+ `incarnation` and `address` for the kit and the page recorder.
+- **Reading a shared file:** `openTelemetryStore(path)` (`@earendil-works/chord/telemetry-sqlite/node`) implements
+ `TelemetryStore` only, on a read-only SQLite connection (`openNodeSqliteDatabase(path, { mode: "read-only" })`:
+ deferred transactions, nothing configured or checkpointed): it never writes, creates or migrates the file, and
+ refuses one without a store or with a store of another version.
## 30. Cost model
diff --git a/packages/chord/docs/guide-plugins-hosts/book.md b/packages/chord/docs/guide-plugins-hosts/book.md
index 6561d2ceb..98287cebf 100644
--- a/packages/chord/docs/guide-plugins-hosts/book.md
+++ b/packages/chord/docs/guide-plugins-hosts/book.md
@@ -817,18 +817,25 @@ answer. Chord's telemetry makes the four steps one **trace**: a tree of **spans*
host recording its own with its own **adapter**, joined by **carriers** (a few strings naming the caller's span) that
ride on the wire with every `hold` and `call`.
-Give each host an adapter. With the app kit that is one option per process; the SQLite store
-(`@earendil-works/chord/telemetry-sqlite`) records settled spans and metrics, and here every process of the app opens
-the same file in the data directory (example 62):
+Give the server a store and the kit does the rest. The SQLite store (`@earendil-works/chord/telemetry-sqlite`)
+records settled spans and metrics in one file in the data directory, and the server is its only writer: every worker
+records in memory and sends its spans and metrics to the server once a second over its link, and a page that records
+(`startPage({ telemetry: true })`) sends its own every few seconds and when hidden (example 62):
```ts
-// start.ts: the server
+// start.ts: the server, its store, and whom spans were for (default: the principal's key)
const telemetry = await createSqliteTelemetry(await openNodeSqliteDatabase(join(dataDir, "telemetry.sqlite")));
-await startServer({ ...options, telemetry }, ctx);
-// worker.ts: each worker, its own recorder on the same file (the path in its environment)
-startWorker({ api, kinds, telemetry: await openTelemetry(process.env.TALLY_TELEMETRY!), onError });
+await startServer({ ...options, telemetry, principalAttributes: (p) => ({ "chord.principal": p.kind }) }, ctx);
+// worker.ts: nothing about telemetry (its own `principalAttributes`, if any)
+startWorker({ api, kinds, onError });
+// the web entry: the page records
+startPage({ ...options, telemetry: true });
```
+A page is untrusted: the server checks each batch's every field (sizes, no control characters, ids printable ASCII),
+keeps only spans under the page's own ids, caps them per page and minute, marks them `chord.host: browser` and stamps
+them with the principal of their link. A click is a trace of its own with `host.root(fn, ctx, { trace: { name } })`.
+
Nothing else changes in plugin code. A method's `ctx.telemetry` is its `chord.method` span, so whatever it calls or
commits is parented under it, on its host or the next one. `ctx.trace(name, attributes, fn)` adds a span of your own:
@@ -842,19 +849,21 @@ add: async (n, ctx) => {
```
The store renders the click's trace across the three hosts (the client's spans, the server's relay, the worker's
-methods and the commit, an event of the method that made it):
+methods and the commit, an event of the method that made it); every span of Chord names the host that recorded it
+(`chord.host`: its address key, `client` without an address), and method and call spans whom they were for
+(`chord.principal`, the host's `principalAttributes` hook):
```
tally.click
- chord.hold type="tally.counter" object="tally.counter(t1)" address.kind="tally" outcome="bound"
- chord.relay type="tally.counter" member="hold" upstream.kind="tally" outcome="ok"
+ chord.hold chord.host="client" type="tally.counter" object="tally.counter(t1)" address.kind="tally" outcome="bound"
+ chord.relay chord.host="server" type="tally.counter" member="hold" upstream.kind="tally" outcome="ok"
→ link to chord.resolve
- chord.call type="tally.counter" member="add" object="tally.counter(t1)" address.kind="tally" outcome="ok"
- chord.relay type="tally.counter" member="#0" upstream.kind="tally" outcome="ok"
- chord.method type="tally.counter" member="add" object="tally.counter(t1)" principal.kind="owner" outcome="ok"
- tally.check n=3
- chord.method type="tally.store" member="add" object="tally.store" principal.kind="worker" outcome="ok"
- · chord.commit documents=1 ops=1 duration=1.9
+ chord.call chord.host="client" type="tally.counter" member="add" object="tally.counter(t1)" address.kind="tally" outcome="ok"
+ chord.relay chord.host="server" type="tally.counter" member="#0" upstream.kind="tally" outcome="ok"
+ chord.method chord.host="tally|t1" type="tally.counter" member="add" object="tally.counter(t1)" principal.kind="owner" chord.principal="{\"kind\":\"owner\"}" outcome="ok"
+ tally.check chord.host="tally|t1" n=3
+ chord.method chord.host="tally|t1" type="tally.store" member="add" object="tally.store" principal.kind="worker" chord.principal="{\"kind\":\"worker\",\"tally\":\"tally|t1\"}" outcome="ok"
+ · chord.commit documents=1 ops=1 rows=0 duration=1.9
```
Work shared by several requests is its own trace, linked rather than nested: the worker's start (`chord.resolve`),
@@ -862,6 +871,8 @@ an object's activation (`chord.activate`, linked to whoever asked first). Routin
`run` and `finish` stay under the carrier its start stored, even when a restarted worker resumes the run. Metrics
(`chord.link.bytes`, `chord.commit.duration`, `chord.calls.in_flight`, ...) are aggregated per 10 s; ids are never their
attributes. Without an adapter (the default), Chord starts no span and sends no carrier: a call costs what it did before.
+A script or a viewer reads the file with `openTelemetryStore(path)` (`@earendil-works/chord/telemetry-sqlite/node`),
+which opens it read-only and offers only the read interface (`findTraces`, `getTrace`, `findMetrics`).
# Reference {.appendix}
diff --git a/packages/chord/docs/guide-plugins-hosts/index.html b/packages/chord/docs/guide-plugins-hosts/index.html
index 7c4184fe6..bab3c7ffe 100644
--- a/packages/chord/docs/guide-plugins-hosts/index.html
+++ b/packages/chord/docs/guide-plugins-hosts/index.html
@@ -734,14 +734,20 @@ method commits. When it is slow, where did the time go? Each host is a different
answer. Chord's telemetry makes the four steps one trace: a tree of spans (named, timed pieces of work), each
host recording its own with its own adapter, joined by carriers (a few strings naming the caller's span) that
ride on the wire with every hold and call.
-
Give each host an adapter. With the app kit that is one option per process; the SQLite store
-(@earendil-works/chord/telemetry-sqlite) records settled spans and metrics, and here every process of the app opens
-the same file in the data directory (example 62):
-
// start.ts: the server
+
Give the server a store and the kit does the rest. The SQLite store (@earendil-works/chord/telemetry-sqlite)
+records settled spans and metrics in one file in the data directory, and the server is its only writer: every worker
+records in memory and sends its spans and metrics to the server once a second over its link, and a page that records
+(startPage({ telemetry: true })) sends its own every few seconds and when hidden (example 62):
+
// start.ts: the server, its store, and whom spans were for (default: the principal's key)const telemetry = awaitcreateSqliteTelemetry(awaitopenNodeSqliteDatabase(join(dataDir, "telemetry.sqlite")));
-awaitstartServer({ ...options, telemetry }, ctx);
-// worker.ts: each worker, its own recorder on the same file (the path in its environment)
-startWorker({ api, kinds, telemetry: awaitopenTelemetry(process.env.TALLY_TELEMETRY!), onError });
+awaitstartServer({ ...options, telemetry, principalAttributes: (p) => ({ "chord.principal": p.kind }) }, ctx);
+// worker.ts: nothing about telemetry (its own `principalAttributes`, if any)
+startWorker({ api, kinds, onError });
+// the web entry: the page records
+startPage({ ...options, telemetry: true });
+
A page is untrusted: the server checks each batch's every field (sizes, no control characters, ids printable ASCII),
+keeps only spans under the page's own ids, caps them per page and minute, marks them chord.host: browser and stamps
+them with the principal of their link. A click is a trace of its own with host.root(fn, ctx, { trace: { name } }).
Nothing else changes in plugin code. A method's ctx.telemetry is its chord.method span, so whatever it calls or
commits is parented under it, on its host or the next one. ctx.trace(name, attributes, fn) adds a span of your own:
add: async (n, ctx) => {
@@ -751,22 +757,26 @@ commits is parented under it, on its host or the next one. ctx.trace(name,
return store.add(n, ctx);
},
The store renders the click's trace across the three hosts (the client's spans, the server's relay, the worker's
-methods and the commit, an event of the method that made it):
+methods and the commit, an event of the method that made it); every span of Chord names the host that recorded it
+(chord.host: its address key, client without an address), and method and call spans whom they were for
+(chord.principal, the host's principalAttributes hook):
Work shared by several requests is its own trace, linked rather than nested: the worker's start (chord.resolve),
an object's activation (chord.activate, linked to whoever asked first). Routines and job phases are roots too; a job's
run and finish stay under the carrier its start stored, even when a restarted worker resumes the run. Metrics
(chord.link.bytes, chord.commit.duration, chord.calls.in_flight, ...) are aggregated per 10 s; ids are never their
-attributes. Without an adapter (the default), Chord starts no span and sends no carrier: a call costs what it did before.
+attributes. Without an adapter (the default), Chord starts no span and sends no carrier: a call costs what it did before.
+A script or a viewer reads the file with openTelemetryStore(path) (@earendil-works/chord/telemetry-sqlite/node),
+which opens it read-only and offers only the read interface (findTraces, getTrace, findMetrics).
A
Reference
diff --git a/packages/chord/docs/tutorial/book.md b/packages/chord/docs/tutorial/book.md
index 941541e05..8f86cbd30 100644
--- a/packages/chord/docs/tutorial/book.md
+++ b/packages/chord/docs/tutorial/book.md
@@ -61,9 +61,9 @@ that step says so and the rest runs.
server, the workers, the pages and the terminals, in place.
8. **Text is rendered per viewer.** A plugin's messages are typed code, a translation is a plugin, each user picks a
language, and servers store keys, never sentences.
-9. **One click is one trace.** Every process records spans to the telemetry store it is given (here one SQLite file
- in the data directory); carriers in the calls' headers join the page's call, the server's relay and the worker's
- method into one trace, which `TelemetryStore` reads back. `ctx.trace` adds spans of your own.
+9. **One click is one trace.** The server writes the telemetry store (here one SQLite file in the data directory); the
+ page and the workers send it their spans; carriers in the calls' headers join the page's click, the server's relay
+ and the worker's method into one trace, which `TelemetryStore` reads back. `ctx.trace` adds spans of your own.
10. **Six traps:** load before use; a reference acts as whoever made it; await what you start; a watch starts at the
current value; the plain `t` has no language of its own; a trace starts where a span starts.
:::
@@ -231,14 +231,15 @@ export function startTodo(
api: API,
kinds: KINDS,
principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", list: address.key }) },
- // The server's spans and metrics; each list's worker opens the same file (`worker.ts`).
+ // The store the server writes: its own spans and metrics, and what the workers and recording pages send.
telemetry: telemetry.store,
+ // Who a span was for: a short key and a user's name (each worker's `startWorker` gets the same hook).
+ principalAttributes,
workers: [
{
kind: v1.List,
entry: fileURLToPath(new URL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
- env: { TODO_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// Who may reach a list's worker: everyone signed in, and the server itself.
authorize: (principal) => principal.kind !== "worker",
@@ -254,22 +255,21 @@ export function startTodo(
}
```
-`workers` is chapter II, `web` chapter IV (its second path, `/traces`, chapter XI), `tui` chapter V, `telemetry`
-chapter XI. `listen` asks for a free port on loopback, as the example does; a real app picks a port. The worker entry
-is a few lines (its telemetry store is chapter XI's too):
+`workers` is chapter II, `web` chapter IV (its second path, `/traces`, chapter XI), `tui` chapter V, `telemetry` and
+`principalAttributes` chapter XI. `listen` asks for a free port on loopback, as the example does; a real app picks a
+port. The worker entry is a few lines (its `principalAttributes` is chapter XI's too):
```ts
// The todo app's worker entry: one process per list. `startWorker` opens the list's store in its directory, connects
-// to the server and loads the plugins' worker facets. The worker records to the app's telemetry store, the file its
-// server named in the environment: its methods called through the server continue the caller's trace.
+// to the server and loads the plugins' worker facets. The worker records its spans in memory and sends them to the
+// server, which stores them: its methods called through the server continue the caller's trace.
import { startWorker } from "../../../src/node/app/index.ts";
-import { openTelemetry } from "./src/telemetry.ts";
+import { principalAttributes } from "./src/principals.ts";
import { API, KINDS } from "./start.ts";
-const telemetry = await openTelemetry(process.env.TODO_TELEMETRY!);
-startWorker({ api: API, kinds: KINDS, telemetry: telemetry.store, onError: (error) => console.error("list:", error) });
+startWorker({ api: API, kinds: KINDS, principalAttributes, onError: (error) => console.error("list:", error) });
```
## Running it
@@ -396,7 +396,6 @@ it on to its parent or to a route (design §3, "Host kinds and addresses"). The
kind: v1.List,
entry: fileURLToPath(new URL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
- env: { TODO_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// Who may reach a list's worker: everyone signed in, and the server itself.
authorize: (principal) => principal.kind !== "worker",
@@ -404,7 +403,7 @@ it on to its parent or to a route (design §3, "Host kinds and addresses"). The
],
```
-`env` names the app's telemetry store for the worker (chapter XI). `authorize` runs for every request routed to a
+`authorize` runs for every request routed to a
list before anything starts; the first request for an address starts
its worker, and a worker idle for five minutes (the default `idleMs`) stops. **Variants** are the member sets a
contract offers: the owner's `Todos` has `clear`, a member's does not. The factory picks one per principal.
@@ -669,7 +668,7 @@ for them:
import type { ComponentType } from "react";
import { defineRegistry } from "../../../../../../src/index.ts";
-export { useHold, useMessages, usePageState, useWatch } from "../../../../../../src/react/index.ts";
+export { useHold, useHost, useMessages, usePageState, useWatch } from "../../../../../../src/react/index.ts";
/** A panel of the page: its name (a key, never shown) and its React component, which shows its own title. */
export type Panel = { readonly name: string; readonly Component: ComponentType };
@@ -702,11 +701,13 @@ void startPage({
connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`),
modules: { "@todo/app/v1": v1, "@todo/app/v1/web": v1web, react: React, "react/jsx-runtime": jsx },
core: [{ name: "@todo/web", facet: core }],
+ // The page records its clicks and the calls under them, and sends them to the server every few seconds.
+ telemetry: true,
render: (host) => createRoot(document.getElementById("root")!).render(h(HostProvider, { host }, h(Page))),
});
```
-The path `/traces` is the owner's trace viewer (chapter XI). `startPage` makes the page's host, connects it to `/chord`
+The path `/traces` is the owner's trace viewer, and `telemetry` what it shows (chapter XI). `startPage` makes the page's host, connects it to `/chord`
(reconnecting after drops), loads the current plugin scripts with the entry's core facets, and keeps them current.
`HostProvider` gives the React tree that host.
@@ -719,6 +720,7 @@ when its items change:
```ts
function TodoList() {
const { t } = useMessages(messages);
+ const host = useHost();
const list = location.hash.slice(1) || "groceries";
const todos = useHold(Todos, { list });
const items = useWatch(todos?.items) ?? [];
@@ -727,8 +729,11 @@ function TodoList() {
const [text, setText] = usePageState(`lists.draft.${list}`, "");
const add = (event: { preventDefault(): void }) => {
event.preventDefault();
- if (todos === undefined || text.trim() === "") return;
- void todos.add(text, context);
+ // An empty todo too: the list's store refuses it, and the click's trace shows where.
+ if (todos === undefined) return;
+ // The click is a trace of its own: a root span of the page, the call to the list's worker under it.
+ const trace = { name: "page.add", attributes: { chars: text.length } };
+ void host.root((ctx) => todos.add(text, ctx), context, { trace }).catch(() => {});
setText("");
};
return h(
@@ -777,7 +782,8 @@ export default defineFacet((_host, self) => {
`messages.ts`, and the facet contributes its catalogs to the page's `Translations`). `useHold` holds a reference while
the component is mounted and releases it on unmount; it is `undefined` on the first render. `useWatch` follows a member. `usePageState` is `useState` whose value the page keeps under a key instead of the
component: chapter IX shows why the half-typed todo needs it. Clicks call methods with `BACKGROUND_CONTEXT`, which never ends, so a click's call
-is not cancelled when the handler returns. `todos?.variant === "owner"` narrows the reference: only the owner's has
+is not cancelled when the handler returns; adding a todo runs in `host.root` (`useHost` gives the page's host), which
+makes the click a trace of its own (chapter XI). `todos?.variant === "owner"` narrows the reference: only the owner's has
`clear`, and TypeScript knows it. The panel's look reads the active theme's CSS variables, with class names prefixed
by the plugin's name (a `*.module.css` file would scope them instead):
@@ -1443,7 +1449,7 @@ the server takes the worker's process lock and reads the store itself. `importDa
replacing a list is a delete, then an import (decision "App kit naming and export"). Registering an imported list in
an index of the app's, if it had one, is the app's.
-# Seeing What Happened {#telemetry lede="One telemetry store for every process, one trace from a click to a commit, a span of your own, and a small viewer for the owner."}
+# Seeing What Happened {#telemetry lede="One telemetry store on the server, one trace from a click in the browser to a commit on a worker, a span of your own, whom each span was for, and a small viewer for the owner."}
Ada says that adding a todo from her phone feels slow some evenings. Her todo crosses three processes: her page, the
server's relay and the list's worker. A log line per process would say what each one did, but not which lines belong
@@ -1452,12 +1458,14 @@ piece of work with attributes), **events** on spans, and **metrics** (counters a
**adapter**, given to `createHost({ telemetry })` and NOOP by default, which decides where they go. A **carrier** in
each call's header names the caller's span, so the spans of one request, recorded by different processes, form one
**trace** (design §29). Mara wants to see them without reading SQL: a small trace viewer would be nice. This chapter
-records, follows one click, adds a span, reads the store from a script, and ends with the viewer.
+stores telemetry in one place, follows a click from the browser to the worker, adds a span, names whom each span was
+for, reads the store from a script, and ends with the viewer.
-## Recording
+## One store, on the server
The app keeps its telemetry in one SQLite file in the data directory, through Chord's store
-(`@earendil-works/chord/telemetry-sqlite`). The app's base code opens it, for any process that asks:
+(`@earendil-works/chord/telemetry-sqlite`), and the server is the only process that writes it. The app's base code opens
+it for the server:
```ts
@@ -1483,89 +1491,89 @@ export async function openTelemetry(file: string): Promise {
```
`createSqliteTelemetry` returns a **recorder**: an adapter that keeps each settled span in memory and writes them in
-one batch a second, one row per span with its events and links inline; metrics are summed per series and 10 seconds;
-rows older than seven days are deleted in the same batch, and when 10,000 spans wait, new ones are dropped and counted.
-Recording never runs SQL. The same object is the store's **read interface**, `TelemetryStore` (below).
+one batch a second, one row per span with its events and links inline and its start and end in microseconds (a
+monotonic clock anchored to the wall clock, so spans of one process keep their order and spacing); metrics are summed
+per series and 10 seconds; rows older than seven days are deleted in the same batch, and when 10,000 spans wait, new
+ones are dropped and counted. Recording never runs SQL. The same object is the store's **read interface**,
+`TelemetryStore` (below).
-Step 1 opened the store before the server (chapter I), and `startTodo` gives it to `startServer`, which passes it to the
-server's host and its HTTP component:
+Step 1 opened the store before the server (chapter I), and `startTodo` gives it to `startServer`:
```ts
- // The server's spans and metrics; each list's worker opens the same file (`worker.ts`).
+ // The store the server writes: its own spans and metrics, and what the workers and recording pages send.
telemetry: telemetry.store,
```
-The kit carries no spans between processes: each process records to the adapter it was given (slice 9, §5). So each
-list's worker opens the same file with a recorder of its own; the server names the file in the worker's environment:
-
-
-```ts
- env: { TODO_TELEMETRY: telemetryFile(dataDir) },
-```
+With a store, the kit does the rest. Every list's worker records in memory and sends its finished spans and metrics to
+the server once a second, over the link it already has; nothing tells a worker where the file is, and the worker's
+entry has no telemetry of its own to set up:
```ts
-const telemetry = await openTelemetry(process.env.TODO_TELEMETRY!);
-startWorker({ api: API, kinds: KINDS, telemetry: telemetry.store, onError: (error) => console.error("list:", error) });
+startWorker({ api: API, kinds: KINDS, principalAttributes, onError: (error) => console.error("list:", error) });
```
-One file for every process is the app's choice. SQLite in WAL mode lets several processes write it, each waiting for
-the others' locks (up to 5 seconds), and every span id is global, so rows of different processes never collide. An app
-with a collector would give each process an adapter that exports there instead; nothing else changes.
+A page records when the app asks it to, in its web entry:
-## One trace across three hosts
-
-A page in a browser cannot record yet: `startPage` takes no telemetry adapter. Step 11 makes the page itself, as Ada's
-client was made in chapter VII: a client host of this process with a recorder of its own on the same file, signed in
-with the owner's cookie from step 1. A click is one span of the page, and the call runs under it:
-
-
+
```ts
-const pageTelemetry = await openTelemetry(telemetryFile(dataDir));
-const page = createHost({
- telemetry: pageTelemetry.store,
- onError: (error) => console.error("page:", error.kind),
-});
-await page.load([], context);
-const pageSocket = new WebSocket(`${url.replace(/^http/, "ws")}/chord`, { headers: { cookie } });
-page.connect(webSocketLink(pageSocket as unknown as WebSocketLike), { principal: { kind: "server" } }, context);
-const pageTodos = await page.get(Todos, { list: "groceries" }, context);
-const click = (text: string) =>
- pageTelemetry.store.startSpan({ name: "page.add" }, (span) => pageTodos.add(text, withTelemetry(span, context)));
-await click("honey");
-await click(" ").catch(() => "refused");
+ // The page records its clicks and the calls under them, and sends them to the server every few seconds.
+ telemetry: true,
```
-`startSpan` runs the callback in a new root span; `withTelemetry(span, context)` makes it the context's span, so the
-call of `add` starts its `chord.call` under it and puts that span's carrier in the call's header. Each recorder writes
-once a second, so the step reads the store until the worker's spans are there, takes the older of the two traces whose
-first span is `page.add` and prints it with pi-telemetry's `renderTrace` (main.ts masks the lengths of time, which
-differ at every run):
+The page keeps its finished spans in memory and sends them every three seconds, and at once when it is hidden (a tab
+switched away may not come back). Both send to an object the kit serves on the server, `WorkerTelemetry` and
+`PageTelemetry`, whose `ingest(batch)` hands the spans to the store; its calls record no span of their own, or every
+batch would write a span about itself. A worker is the server's own child: its batches are written as they are,
+marked with its host. A page is anyone's: sign-in and the link's limits apply as for any call, and the server checks
+everything (next section).
-
+## One trace from the browser
+
+The lists plugin's panel starts a trace where Ada's click starts: in its form's `submit`, a root span of the page's host
+with the call under it:
+
+
+```ts
+ // The click is a trace of its own: a root span of the page, the call to the list's worker under it.
+ const trace = { name: "page.add", attributes: { chars: text.length } };
+ void host.root((ctx) => todos.add(text, ctx), context, { trace }).catch(() => {});
+```
+
+`host.root(fn, context, { trace })` runs `fn` in a new root span of the host's adapter, `ctx.telemetry` that span, so
+the call of `add` starts its `chord.call` under it and puts that span's carrier in the call's header (`useHost` gives a
+component its page's host). Without a recording adapter it starts nothing. `screenshots.ts` signs the owner in to the
+page in Chromium, types honey and submits, then submits again with nothing typed; it reads the store until the
+worker's spans of both clicks are there and prints honey's trace with pi-telemetry's `renderTrace` (ids and lengths of
+time masked, which differ at every run):
+
+
```text
-11. adding honey from the page:
- page.add
- chord.call type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="ok"
- chord.relay type="todo.todos" member="#0" upstream.kind="list" outcome="ok"
- chord.method type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" outcome="ok"
- chord.method type="todo.store" member="add" object="todo.store" principal.kind="worker" outcome="ok"
- · chord.commit documents=0 ops=0 duration=
- todos.validate chars=5
+adding honey from the page:
+ page.add chord.host="browser" chars=5 chord.principal="owner"
+ chord.call chord.host="browser" type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="ok" chord.principal="owner"
+ chord.relay chord.host="server" type="todo.todos" member="#0" upstream.kind="list" outcome="ok"
+ chord.method chord.host="list|groceries" type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" chord.principal="owner" outcome="ok"
+ chord.method chord.host="list|groceries" type="todo.store" member="add" object="todo.store" principal.kind="worker" chord.principal="worker:list|groceries" outcome="ok"
+ · chord.commit documents=0 ops=0 rows=1 duration=
+ todos.validate chord.host="list|groceries" chars=5
```
-Three processes recorded these six spans. The page recorded `page.add` and `chord.call`, the consumer's call until its
-reply. The server recorded `chord.relay`: it forwarded the call to the list's worker without knowing member names
-(`#0` is the method's index). The worker recorded the rest: `chord.method` for the owner's `Todos` view, then the
-view's call of the store, which is a method span too, by `worker`, because the store acts as its host (chapter II).
-The commit is an event on the span its transaction ran under, and the renderer prints a span's events before its
-children. Its counts are documents and their ops; a commit of table rows counts neither, so this one says 0 and 0.
+Three processes recorded these six spans, and the server wrote them all. Each span says which host recorded it:
+`chord.host` is the host's address key (`server`, `list|groceries`), and `browser` for what a page sent. The page
+recorded `page.add` and `chord.call`, the consumer's call until its reply. The server recorded `chord.relay`: it
+forwarded the call to the list's worker without knowing member names (`#0` is the method's index). The worker recorded
+the rest: `chord.method` for the owner's `Todos` view, then the view's call of the store, which is a method span too,
+by `worker`, because the store acts as its host (chapter II). The commit is an event on the span its transaction ran
+under, and the renderer prints a span's events before its children. It counts the documents it changed, their ops and
+the table rows its writes changed: the store keeps todos in a table, so one row and no documents. `chord.principal`
+says whom a span was for (below).
-:::sequence animate | One click, one trace: each process starts its spans with its own recorder, and the carrier in each call's header names the parent span on the other side.
-actor page "page" "recorder 1"
-actor server "server" "recorder 2"
-actor worker "list|groceries" "recorder 3"
+:::sequence animate | One click, one trace: each process starts its spans with its own adapter, the carrier in each call's header names the parent span on the other side, and the page and the worker send their spans to the server, which writes them.
+actor page "page" "browser"
+actor server "server" "the store"
+actor worker "list|groceries" "worker"
page -> page: page.add, chord.call
page => server: call add, carrier of chord.call
server -> server: chord.relay
@@ -1573,25 +1581,49 @@ server => worker: call #0, carrier of chord.relay
worker -> worker: chord.method ×2, todos.validate, commit
worker --> server: reply
server --> page: reply
+worker => server: ingest: its spans (every second)
+page => server: ingest: its spans (every 3 s, or hidden)
:::
The carrier is a field of the `call` and `hold` headers, at most 8 keys of at most 256 characters each. The receiver
reads it with its own adapter; an absent or malformed one gives a new root, never an error. With the NOOP adapter
nothing is injected and no span starts: a span site costs one check (§30).
+What a page sends, the server treats as untrusted. A batch is checked by its schema: at most 100 spans, every field
+bounded (names 128 characters, 32 attributes of at most 256 characters, 16 events, 8 links, a 500-character error
+message), ids printable ASCII and no control characters anywhere; a batch outside it is refused whole. A page's span
+ids are `browser::`, and the server takes only spans whose id, trace and parent carry the
+calling page's own prefix: a page can neither write into another sender's trace nor pose as a server span, and the
+store never replaces a stored span. A page may send 1,200 spans a minute; past that they are dropped and counted in
+`ingest`'s answer. The server marks each span `chord.host: browser` and stamps it with the principal of the link it came
+over: what a page says about itself counts for nothing.
+
The trace holds the click and nothing else. Work that serves many requests is its own trace, **linked** to the span that
first needed it: a worker's start (`chord.resolve`), an object's activation (`chord.activate`). A link is a reference to
a span of another trace; the renderer prints it and never follows it.
## Trap: a trace starts where a span starts
-The page took its reference to `Todos` before the click, under `BACKGROUND_CONTEXT`, and that hold is in no trace of
-the page: **a context without a recording span records nothing.** A host gives such work no span of its own, so its
-calls start no `chord.call` and send no carrier, and the provider's `chord.method` becomes a root of its own trace
+The panel took its reference to `Todos` before the click (`useHold`), under `BACKGROUND_CONTEXT`, and that hold is in no
+trace of the page: **a context without a recording span records nothing.** A host gives such work no span of its own,
+so its calls start no `chord.call` and send no carrier, and the provider's `chord.method` becomes a root of its own trace
(the server's relay of the page's hold is one: `chord.relay` with `member="hold"`). So were the server's calls in step 2:
-on the worker each is a root `chord.method`. Start a span where the work begins, a click handler or a job, and pass it
-down with `withTelemetry`. The kit's HTTP component does this for every request: a `chord.http` span (method, route
-pattern, status), with the route's `ctx` under it.
+on the worker each is a root `chord.method`. Start a span where the work begins, a click handler or a job
+(`host.root(fn, context, { trace })`), and pass its `ctx` down. The kit's HTTP component does this for every request: a
+`chord.http` span (method, route pattern, status), with the route's `ctx` under it. Step 11 adds honey from the command
+line, as step 3 did; the request is the trace's root, the server's call to the worker under it:
+
+
+```text
+11. adding honey from the command line:
+ chord.http chord.host="server" method="POST" route="POST /api/lists/:list" status=200
+ chord.hold chord.host="server" type="todo.todos" object="todo.todos(groceries)" address.kind="list" outcome="bound"
+ chord.call chord.host="server" type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" chord.principal="owner" outcome="ok"
+ chord.method chord.host="list|groceries" type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" chord.principal="owner" outcome="ok"
+ chord.method chord.host="list|groceries" type="todo.store" member="add" object="todo.store" principal.kind="worker" chord.principal="worker:list|groceries" outcome="ok"
+ · chord.commit documents=0 ops=0 rows=1 duration=
+ todos.validate chord.host="list|groceries" chars=5
+```
## A span of your own
@@ -1606,26 +1638,63 @@ The store's `add` checks the text in a span of its own:
```
`ctx.trace(name, attributes, fn)` starts a child of the method's span and gives `fn` a child `ctx` with the same
-principal and `get`, so calls made inside are parented under it. It is the last line of the trace above:
-`todos.validate chars=5`. Its attributes are small, flat values; the todo's text is data and stays out of telemetry,
-its length is enough. When `fn` throws, the span fails and the error passes through as it was: the caller still gets
-`invalid/args`. A check this small does not need a span of its own; a lookup, a model request or a file read does.
+principal and `get`, so calls made inside are parented under it, and names its host as Chord's spans do. It is the last
+line of both traces above: `todos.validate chord.host="list|groceries" chars=5`. Its attributes are small, flat values;
+the todo's text is data and stays out of telemetry, its length is enough. When `fn` throws, the span fails and the error
+passes through as it was: the caller still gets `invalid/args`. A check this small does not need a span of its own; a
+lookup, a model request or a file read does.
+
+## Whom a span was for
+
+Method and call spans record whom they were for: by default `chord.principal`, the principal's key (its canonical
+JSON). An app may record other attributes instead, through a hook each host takes (`principalAttributes`): the todo
+app's records a short key and, for a user, the name she signed in with:
+
+
+```ts
+export function principalAttributes(principal: TodoPrincipal): Readonly> {
+ let attributes = recorded.get(principal);
+ if (attributes === undefined) {
+ attributes =
+ principal.kind === "user"
+ ? { "chord.principal": `user:${principal.id}`, "user.name": principal.name }
+ : { "chord.principal": principal.kind === "worker" ? `worker:${principal.list}` : principal.kind };
+ recorded.set(principal, attributes);
+ }
+ return attributes;
+}
+```
+
+Chord calls the hook for every span it records with a principal, so it must be synchronous and cheap; any caching is
+the hook's own (here a `WeakMap` by principal: a link stamps one principal object on all its calls). `startTodo` gives
+it to the server (`principalAttributes` beside `telemetry`) and the worker's entry to `startWorker` (above): each host
+has its own, the same function. The server also stamps a page's spans with it, for the principal of the link the batch
+came over: `page.add` above was the owner's. Only what the hook returns is recorded, never other app data, and the
+attributes are for the owner's eyes: the store, like the viewer below, is the owner's.
## Reading
The store is read through `TelemetryStore`, pi-telemetry's read interface modeled on Jaeger's: `findTraces` (complete
traces, newest first, filtered by their first span's name, attributes, errors or a time window, paged by a cursor),
`getTrace(id)` and `findMetrics` (a metric's rows, one per series and 10 seconds, with count, sum, min and max).
-`trace.ts` is a script for a data directory, the app running or not:
+`trace.ts` is a script for a data directory, the app running or not. It opens the file with Chord's read-only store,
+`openTelemetryStore` (`@earendil-works/chord/telemetry-sqlite/node`), which implements only the read interface and
+opens SQLite read-only: it never writes the file the server writes, never creates or migrates it, and refuses a file
+without a store or with a store of another version.
```ts
+// The shared file, read-only: the app's processes may be writing it meanwhile.
+const store = await openTelemetryStore(telemetryFile(dataDir));
+try {
if (id !== undefined) {
- const trace = await telemetry.store.getTrace(id);
+ const trace = await store.getTrace(id);
console.log(trace === undefined ? `no trace ${id}` : renderTrace(trace.spans));
} else {
- // Complete traces, newest first; a trace's first span (its root, by start time) names it.
- const { traces } = await telemetry.store.findTraces({ name: values.name, limit: 10 });
+ // Complete traces, newest first; a trace's first span (its root, by start time) names it, and the app's
+ // `principalAttributes` hook named whom each span was for.
+ const attributes = values.principal === undefined ? undefined : { "chord.principal": values.principal };
+ const { traces } = await store.findTraces({ name: values.name, attributes, limit: 10 });
for (const { traceId, spans } of traces) {
const start = Math.min(...spans.map((span) => span.startedMs));
const length = Math.max(...spans.map((span) => span.endedMs)) - start;
@@ -1634,7 +1703,7 @@ traces, newest first, filtered by their first span's name, attributes, errors or
}
// Metrics are stored per series and 10 s: a series is one set of attributes, here the direction.
for (const direction of ["sent", "received"]) {
- const rows = await telemetry.store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
+ const rows = await store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
const bytes = rows.reduce((sum, row) => sum + row.sum, 0);
const messages = rows.reduce((sum, row) => sum + row.count, 0);
console.log(`chord.link.bytes ${direction}: ${bytes} bytes in ${messages} messages`);
@@ -1642,62 +1711,63 @@ traces, newest first, filtered by their first span's name, attributes, errors or
}
```
-Step 11 runs it twice from a process of its own: the newest `page.add` traces, then the refused one by its id (ids,
-lengths and counts masked):
+`screenshots.ts` runs it twice from a process of its own while the app runs: the newest `page.add` traces with a span
+for the owner, then the refused click by its id (ids, lengths and counts masked):
-
+
```text
- trace.ts --name page.add:
+trace.ts --name page.add --principal owner:
page.add 6 spans error
page.add 6 spans
chord.link.bytes sent: bytes in messages
chord.link.bytes received: bytes in messages
- trace.ts :
- page.add error ChordError: a todo needs text
- chord.call type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="error" error call_failed: a todo needs text
- chord.relay type="todo.todos" member="#0" upstream.kind="list" outcome="error" error relay_failed: a todo needs text
- chord.method type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" outcome="error" error method_failed: a todo needs text
- chord.method type="todo.store" member="add" object="todo.store" principal.kind="worker" outcome="error" error method_failed: a todo needs text
- todos.validate chars=0 error trace_failed: a todo needs text
+trace.ts :
+ page.add chord.host="browser" chars=0 chord.principal="owner" error trace_failed: a todo needs text
+ chord.call chord.host="browser" type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="error" chord.principal="owner" error call_failed: a todo needs text
+ chord.relay chord.host="server" type="todo.todos" member="#0" upstream.kind="list" outcome="error" error relay_failed: a todo needs text
+ chord.method chord.host="list|groceries" type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" chord.principal="owner" outcome="error" error method_failed: a todo needs text
+ chord.method chord.host="list|groceries" type="todo.store" member="add" object="todo.store" principal.kind="worker" chord.principal="worker:list|groceries" outcome="error" error method_failed: a todo needs text
+ todos.validate chord.host="list|groceries" chars=0 error trace_failed: a todo needs text
```
A trace's id is its root span's. The refused add failed on every span of its path, each with a fixed category
-(`call_failed`, `relay_failed`, `method_failed`, `trace_failed`) and the message cut at 500 characters; a cancelled call
-is not an error. It has no commit: nothing was written. `chord.link.bytes` counts the encoded messages of every host's
-links, each message once where it was sent and once where it was received. A metric carries no ids, only attributes
-with few values (`direction`), so its series stay few. A running recorder writes a metric's current 10 seconds only
-once they are over, or when it closes: `trace.ts` sees the page's bytes, whose recorder closed, and the server's and
-workers' of the intervals already over. The script is a recorder too, which records nothing; Chord ships only the
-store, so reads go through a recorder of the file.
+(`trace_failed`, `call_failed`, `relay_failed`, `method_failed`) and the message cut at 500 characters; a cancelled call
+is not an error. It has no commit: nothing was written. `chord.link.bytes` counts the encoded messages of the server's
+and the workers' links (a page sends no metrics), each message once where it was sent and once where it was received.
+A metric carries no ids, only attributes with few values (`direction`), so its series stay few. A metric's current 10
+seconds are sent and written only once they are over, or when a process closes.
## A trace viewer
Two halves make the viewer: routes on the server that read the store, and a page that shows what they answer. Server
code of an app is either the start script or a plugin, and routes come from plugins (chapter III), so the routes are a
-small plugin, `plugins/traces/`. Its server facet serves one host-lifetime object, which opens the store's file as a
-native and contributes the routes while it is live:
+small plugin, `plugins/traces/`. Its server facet serves one host-lifetime object, which opens the store's file
+read-only as a native and contributes the routes while it is live:
```ts
host.serve(Traces, (_params, self) => {
const info = self.get(AppInfo, {});
- // The server records to the same file with its own recorder; this one only reads.
+ // The server records to the same file with its own recorder; this object only reads it.
const telemetry = self.open(
- async () => openTelemetry(telemetryFile(info.dataDir.value)),
- (opened) => opened.close(),
+ async () => openTelemetryStore(telemetryFile(info.dataDir.value)),
+ (store) => store.close(),
);
self.contribute(Http, {
routes: {
- "GET /api/traces": async (_request, ctx) => {
+ "GET /api/traces": async (request, ctx) => {
ownerOnly(ctx);
- const { traces } = await telemetry.value.store.findTraces({ limit: RECENT });
+ // `?principal=user:u1`: the traces with a span for that principal (`principalAttributes`).
+ const principal = request.query.get("principal");
+ const attributes = principal === null ? undefined : { "chord.principal": principal };
+ const { traces } = await telemetry.value.findTraces({ limit: RECENT, attributes });
return { kind: "json", value: traces.map(({ traceId, spans }) => summaryOf(traceId, spans)) };
},
"GET /api/traces/:id": async (request, ctx) => {
ownerOnly(ctx);
- const trace = await telemetry.value.store.getTrace(request.params.id!);
+ const trace = await telemetry.value.getTrace(request.params.id!);
if (trace === undefined) return { kind: "error", status: 404 };
- return { kind: "json", value: { id: trace.traceId, spans: rowsOf(trace.spans) } };
+ return { kind: "json", value: { id: trace.traceId, spans: rowsOf(clamped(trace.spans)) } };
},
},
});
@@ -1706,10 +1776,10 @@ native and contributes the routes while it is live:
```
The kit gives a plugin the server's process values as objects: `AppInfo` names the data directory, and the API's server
-module re-exports it with the app's `openTelemetry`. `self.open` declares a native, opened once `AppInfo` is live and
-closed when the object ends; a reload of the plugin restarts the object and opens the file anew. An object may
-contribute when it lives as long as its host (`local`, empty params; decision R21). Spans name who called what, so
-only the owner reads them:
+module re-exports it with `telemetryFile` and Chord's `openTelemetryStore`. `self.open` declares a native, opened once
+`AppInfo` is live and closed when the object ends; a reload of the plugin restarts the object and opens the file anew.
+An object may contribute when it lives as long as its host (`local`, empty params; decision R21). Spans name who called
+what, so only the owner reads them:
```ts
@@ -1718,20 +1788,27 @@ function ownerOnly(ctx: Ctx): void {
}
```
-A route's principal is `undefined` only on a public route, which these are not; a signed-in member gets 403. `rowsOf`
-orders a trace's spans as a tree, parents before their children, and turns attributes, events and links into text. The
-spans name no host; the SQLite store's span ids begin with the id of the recorder, one per process, and the viewer
-calls each recorder a host:
+A route's principal is `undefined` only on a public route, which these are not; a signed-in member gets 403. The list
+filters by whom the traces were for (`?principal=`), and shows each with its user's name or principal key. `rowsOf`
+orders a trace's spans as a tree, parents before their children, each row's host its span's `chord.host`, and turns
+attributes, events and links into text. A page's spans carry the page's clock, which may be off by seconds against the
+server's, so a span of the page could show starting after the server span under it. Before drawing, `clamped` moves
+the page's spans of the trace by the least amount that puts every server span under a page span inside it
+(`plugins/traces/clock.ts`, with its own test):
-
+
```ts
-function processOf(span: StoredSpan): string {
- return span.id.slice(0, span.id.lastIndexOf(":"));
+export function clamped(spans: readonly StoredSpan[]): readonly StoredSpan[] {
+ const skew = skewOf(spans);
+ if (skew === 0) return spans;
+ return spans.map((span) =>
+ hostOf(span) === "browser" ? { ...span, startedMs: span.startedMs + skew, endedMs: span.endedMs + skew } : span,
+ );
}
```
-That is a detail of this store, not of the read interface (see the epilogue). The page is the app's own page at a
-second path: `start.ts` serves it at `/` and `/traces` (chapter I), and the web entry picks the viewer by the path:
+The page is the app's own page at a second path: `start.ts` serves it at `/` and `/traces` (chapter I), and the web
+entry picks the viewer by the path:
```ts
@@ -1740,9 +1817,9 @@ second path: `start.ts` serves it at `/` and `/traces` (chapter I), and the web
```
`TraceViewer` (`src/web/traces.ts`) fetches the list and the trace chosen in the URL's fragment, and draws each span
-with its host, its length as a bar on the trace's time line, its attributes and events, a failed span in the theme's
-error colour, and each link as the ids it refers to, never fetched. The bars are positioned with inline styles, which
-the page's CSP allows only because React sets them through the DOM:
+with its host's name, its length as a bar on the trace's time line, its attributes and events, a failed span in the
+theme's error colour, and each link as the ids it refers to, never fetched. The bars are positioned with inline
+styles, which the page's CSP allows only because React sets them through the DOM:
```ts
@@ -1752,8 +1829,7 @@ the page's CSP allows only because React sets them through the DOM:
};
```
-`screenshots.ts` adds honey from a recording page and refuses a blank todo, as step 11 does, then opens the viewer on
-honey's trace:
+After the two clicks, `screenshots.ts` opens the viewer on honey's trace, then types `user:u1` into its filter:
```ts
@@ -1766,15 +1842,22 @@ honey's trace:
```text
wrote docs/tutorial/screens/traces.png
the viewer shows page.add: 6 spans on 3 hosts
+wrote docs/tutorial/screens/traces-ada.png
+the viewer for user:u1 lists chord.method for ada
```
-:::figure wide | The owner's trace viewer at `/traces`, in the Harbor theme: the newest traces (the refused add marked in red, the rest the app's own work: relays, activations, HTTP requests), and honey's trace as a tree across the page, the server and the worker. Starts are stored in whole milliseconds, so the bars of spans this short shift by up to one.
+:::figure wide | The owner's trace viewer at `/traces`, in the Harbor theme: the newest traces (the refused click marked in red, each with whom it was for; the rest the app's own work: relays, activations, HTTP requests), and honey's trace as a tree across the page (`browser`), the server and the worker (`list|groceries`), all written by the server. Starts are stored to the microsecond, so even bars of a fraction of a millisecond line up.

:::
+:::figure wide | The traces for Ada (`user:u1`): the coffee `screenshots.ts` adds at its start as her (a root of the server acting as her), recorded by the list's worker; the app's hook named her `ada`.
+
+:::
+
The list is mostly the app's own work: each HTTP request, relayed hold and activation is a trace of its own, as the
-trap above says. A real viewer would filter by name (`findTraces({ name })`) or by error (`errors: true`), both of
-which the read interface has; this one is about 270 lines of TypeScript in two files and 20 of CSS, and no Chord API was added for it.
+trap above says. A real viewer would filter by name too (`findTraces({ name })`) or by error (`errors: true`), both of
+which the read interface has; this one is about 330 lines of TypeScript in three files and 20 of CSS, on Chord's
+read-only store.
# Testing One Plugin {#testing lede="testPlugin builds a plugin as production does and loads it alone on a host of its own."}
@@ -1997,36 +2080,37 @@ chapter I), plugins could not throw Chord's `invalid` (Chord exports `invalid` a
live window over a table needed a one-time read in `self.open`, which made the store restart at every reload of its
plugin (now `self.query`, a live query Chord runs, chapters II and IX).
-**Found while writing the telemetry chapter, still open.** A page cannot record: `startPage` takes no telemetry
-adapter, so chapter XI's page is a client host in Node, and a browser's click starts its trace at the server's relay.
-`chord.commit` counts documents and their ops only, so a commit that wrote table rows reports `documents=0 ops=0`.
-Spans name no host, and the viewer tells processes apart by the SQLite store's id format (`:`), which
-the read interface does not promise. The store keeps a span's start in whole milliseconds (`Date.now()`) and its
-length in fractions (`performance.now()`), so on a trace of a few milliseconds the viewer's bars shift by up to one. And
-reading needs a recorder: `chord/telemetry-sqlite` exports no read-only
-store, so a script or a viewer opens a recorder that records nothing.
+**Found while writing the telemetry chapter, and fixed in Chord since.** `chord.commit` counted documents and their
+ops only, so a commit that wrote table rows reported `documents=0 ops=0` (it counts `rows` now). Spans named no host,
+and the viewer told processes apart by the SQLite store's id format (every Chord span carries `chord.host` now). The
+store kept a span's start in whole milliseconds, so the viewer's bars shifted by up to one (starts are microseconds
+now, the store's version 2, which a recorder migrates its file to). Reading needed a recorder that recorded nothing
+(`openTelemetryStore` reads read-only). A page could not record, so the chapter's click was a client host in Node
+standing in for the page (`startPage({ telemetry: true })` now, its spans sent to the server). And every process wrote
+the one file itself, each worker told its path (the server is the store's only writer now, the workers and pages
+sending it their spans), and spans said nothing of whom they were for (`principalAttributes`).
# The App's Files {.appendix}
| File | Kind | What it is |
| --- | --- | --- |
-| `src/principals.ts` | base | the principal type |
+| `src/principals.ts` | base | the principal type, `principalAttributes` (whom a span was for) |
| `package.json` | base | the API as the workspace package `@todo/app`, its `exports` |
| `src/api/v1/index.ts` | base, every kind | `@todo/app/v1`: Chord's part (with `invalid`, `forbidden`, `defineMessages`, `Translations`), `defineFacet`, `List` |
-| `src/api/v1/server.ts` | base, server | `Http`, the ephemeral `login`, `AppInfo`, `openTelemetry` |
-| `src/api/v1/web.ts` | base, web | `Panels`, `useHold`, `useMessages`, `usePageState`, `useWatch` |
+| `src/api/v1/server.ts` | base, server | `Http`, the ephemeral `login`, `AppInfo`, `telemetryFile`, `openTelemetryStore` |
+| `src/api/v1/web.ts` | base, web | `Panels`, `useHold`, `useHost`, `useMessages`, `usePageState`, `useWatch` |
| `src/api/v1/tui.ts` | base, tui | `Screens` |
-| `start.ts`, `worker.ts` | base | `startServer`, `startWorker`, each with the telemetry store |
-| `src/telemetry.ts` | base | the telemetry store's file in the data directory, `openTelemetry` |
-| `src/web/main.ts`, `src/web/style.css` | base, web entry | `startPage`, the panels, the viewer at `/traces` |
+| `start.ts`, `worker.ts` | base | `startServer` with the telemetry store, `startWorker`, each with `principalAttributes` |
+| `src/telemetry.ts` | base | the telemetry store's file in the data directory, `openTelemetry` (the server's) |
+| `src/web/main.ts`, `src/web/style.css` | base, web entry | `startPage` (recording), the panels, the viewer at `/traces` |
| `src/web/traces.ts` | base, web entry | the owner's trace viewer |
| `src/tui/main.ts` | base, terminal entry | `startTerminalRuntime`, the screens |
| `plugins/lists/` | plugin | contract (`Item`, `Notice`, `Todos`), messages (English, German), worker (table, store, views), server (routes), web (panel), tui (screen), its test |
| `plugins/french/` | plugin | French for the lists plugin, for pages and terminals |
| `plugins/login.server.ts` | single-file plugin | the kit's ephemeral login |
-| `plugins/traces/` | plugin | the trace viewer's routes over the telemetry store (server), the owner's only |
+| `plugins/traces/` | plugin | the trace viewer's routes over the telemetry store (server), the owner's only; the page's clock moved onto the server's (`clock.ts`, its test) |
| `later/harbor/` | plugin, added in step 6 | the Harbor theme for pages and terminals |
-| `main.ts`, `screenshots.ts`, `tutorial.test.ts` | example | the steps, the screenshots, the checks |
+| `main.ts`, `screenshots.ts`, `tutorial.test.ts` | example | the steps, the screenshots (with the page's traced click), the checks |
| `trace.ts` | example | a data directory's newest traces, one trace, a metric |
# Reference {.appendix}
@@ -2034,7 +2118,7 @@ store, so a script or a viewer opens a recorder that records nothing.
| Call | Chapter | What it does |
| --- | --- | --- |
| `startServer
({ name, dataDir, root, listen, api, kinds, principal, workers, web, tui, onError }, ctx)` | I | the app's server, with the kit |
-| `startWorker({ api, kinds, onError })` | I | a worker's entry |
+| `startWorker({ api, kinds, principalAttributes, onError })` | I, XI | a worker's entry |
| `app.ownerLink()`, `app.ownerTokenFile` | I, III | the owner's sign-in, for a browser and for local clients |
| `hostKind(name, params)`, `remote(name, { version, params, hostAddress, variants })`, `local(name, { version, params, members })` | II | host kinds and contracts |
| `host.serve(Type, factory)`; factory `self`: `table`, `query`, `derive`, `tx`, `get`, `contribute` | II | an object's parts |
@@ -2051,8 +2135,8 @@ store, so a script or a viewer opens a recorder that records nothing.
| `MessagesOf`, `LocaleChoice` | VIII | a translation plugin, the user's language |
| `app.plugins.state`, `app.applyChanges(ctx)` | VI, IX | marks and rollouts |
| `app.exportData(address, path, ctx)`, `app.importData(path, address, ctx)` | X | moving a worker's data |
-| `createSqliteTelemetry(db)`, `startServer({ telemetry })`, `startWorker({ telemetry })`, `createHost({ telemetry })` | XI | recording to the telemetry store |
-| `adapter.startSpan({ name }, fn)`, `withTelemetry(span, ctx)`, `ctx.trace(name, attributes, fn)` | XI | spans of your own |
-| `store.findTraces({ name, attributes, errors, startMin, startMax, limit, cursor })`, `store.getTrace(id)`, `store.findMetrics({ name, attributes })`, `renderTrace(spans)` | XI | reading traces and metrics |
+| `createSqliteTelemetry(db)`, `startServer({ telemetry, principalAttributes })`, `startPage({ telemetry: true })`, `createHost({ telemetry, principalAttributes })` | XI | the telemetry store on the server, pages and workers sending it their spans |
+| `host.root(fn, ctx, { trace })`, `ctx.trace(name, attributes, fn)` | XI | spans of your own |
+| `openTelemetryStore(path)`, `store.findTraces({ name, attributes, errors, startMin, startMax, limit, cursor })`, `store.getTrace(id)`, `store.findMetrics({ name, attributes })`, `renderTrace(spans)` | XI | reading traces and metrics |
| `AppInfo`, `self.open(open, close)` | XI | the data directory, a native (the viewer's store) |
| `testPlugin(dir, { kind, api, modules, address, principal, stubs }, ctx)` | XII | one plugin alone |
diff --git a/packages/chord/docs/tutorial/index.html b/packages/chord/docs/tutorial/index.html
index 7a1569a3b..7b92fdfce 100644
--- a/packages/chord/docs/tutorial/index.html
+++ b/packages/chord/docs/tutorial/index.html
@@ -176,9 +176,9 @@ plugins contribute, one plugin (or the entry) composes.
server, the workers, the pages and the terminals, in place.
Text is rendered per viewer. A plugin's messages are typed code, a translation is a plugin, each user picks a
language, and servers store keys, never sentences.
-
One click is one trace. Every process records spans to the telemetry store it is given (here one SQLite file
-in the data directory); carriers in the calls' headers join the page's call, the server's relay and the worker's
-method into one trace, which TelemetryStore reads back. ctx.trace adds spans of your own.
+
One click is one trace. The server writes the telemetry store (here one SQLite file in the data directory); the
+page and the workers send it their spans; carriers in the calls' headers join the page's click, the server's relay
+and the worker's method into one trace, which TelemetryStore reads back. ctx.trace adds spans of your own.
Six traps: load before use; a reference acts as whoever made it; await what you start; a watch starts at the
current value; the plain t has no language of its own; a trace starts where a span starts.
@@ -315,14 +315,15 @@ facet kinds (which files a plugin may have), and the principals:
api: API,
kinds: KINDS,
principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", list: address.key }) },
- // The server's spans and metrics; each list's worker opens the same file (`worker.ts`).
+ // The store the server writes: its own spans and metrics, and what the workers and recording pages send.
telemetry: telemetry.store,
+ // Who a span was for: a short key and a user's name (each worker's `startWorker` gets the same hook).
+ principalAttributes,
workers: [
{
kind: v1.List,
entry: fileURLToPath(newURL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
- env: { TODO_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// Who may reach a list's worker: everyone signed in, and the server itself.
authorize: (principal) => principal.kind !== "worker",
@@ -336,20 +337,19 @@ facet kinds (which files a plugin may have), and the principals:
context,
);
}
-
workers is chapter II, web chapter IV (its second path, /traces, chapter XI), tui chapter V, telemetry
-chapter XI. listen asks for a free port on loopback, as the example does; a real app picks a port. The worker entry
-is a few lines (its telemetry store is chapter XI's too):
+
workers is chapter II, web chapter IV (its second path, /traces, chapter XI), tui chapter V, telemetry and
+principalAttributes chapter XI. listen asks for a free port on loopback, as the example does; a real app picks a
+port. The worker entry is a few lines (its principalAttributes is chapter XI's too):
// The todo app's worker entry: one process per list. `startWorker` opens the list's store in its directory, connects
-// to the server and loads the plugins' worker facets. The worker records to the app's telemetry store, the file its
-// server named in the environment: its methods called through the server continue the caller's trace.
+// to the server and loads the plugins' worker facets. The worker records its spans in memory and sends them to the
+// server, which stores them: its methods called through the server continue the caller's trace.import { startWorker } from"../../../src/node/app/index.ts";
-import { openTelemetry } from"./src/telemetry.ts";
+import { principalAttributes } from"./src/principals.ts";
import { API, KINDS } from"./start.ts";
-const telemetry = awaitopenTelemetry(process.env.TODO_TELEMETRY!);
-startWorker({ api: API, kinds: KINDS, telemetry: telemetry.store, onError: (error) => console.error("list:", error) });
Step 1 of main.ts starts the app, tries the page without signing in, then does what a browser does with the owner
link:
@@ -453,13 +453,12 @@ it on to its parent or to a route (design §3, "Host kinds and addresses&qu
kind: v1.List,
entry: fileURLToPath(newURL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
- env: { TODO_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// Who may reach a list's worker: everyone signed in, and the server itself.
authorize: (principal) => principal.kind !== "worker",
},
],
-
env names the app's telemetry store for the worker (chapter XI). authorize runs for every request routed to a
+
authorize runs for every request routed to a
list before anything starts; the first request for an address starts
its worker, and a worker idle for five minutes (the default idleMs) stops. Variants are the member sets a
contract offers: the owner's Todos has clear, a member's does not. The factory picks one per principal.
@@ -666,7 +665,7 @@ for them:
importtype { ComponentType } from"react";
import { defineRegistry } from"../../../../../../src/index.ts";
-export { useHold, useMessages, usePageState, useWatch } from"../../../../../../src/react/index.ts";
+export { useHold, useHost, useMessages, usePageState, useWatch } from"../../../../../../src/react/index.ts";
/** A panel of the page: its name (a key, never shown) and its React component, which shows its own title. */exporttypePanel = { readonly name: string; readonlyComponent: ComponentType };
@@ -695,9 +694,11 @@ for them:
connect: () => newWebSocket(`${location.origin.replace(/^http/, "ws")}/chord`),
modules: { "@todo/app/v1": v1, "@todo/app/v1/web": v1web, react: React, "react/jsx-runtime": jsx },
core: [{ name: "@todo/web", facet: core }],
+ // The page records its clicks and the calls under them, and sends them to the server every few seconds.
+ telemetry: true,
render: (host) => createRoot(document.getElementById("root")!).render(h(HostProvider, { host }, h(Page))),
});
-
The path /traces is the owner's trace viewer (chapter XI). startPage makes the page's host, connects it to /chord
+
The path /traces is the owner's trace viewer, and telemetry what it shows (chapter XI). startPage makes the page's host, connects it to /chord
(reconnecting after drops), loads the current plugin scripts with the entry's core facets, and keeps them current.
HostProvider gives the React tree that host.
The web facet
@@ -706,6 +707,7 @@ when its items change:
functionTodoList() {
const { t } = useMessages(messages);
+ const host = useHost();
const list = location.hash.slice(1) || "groceries";
const todos = useHold(Todos, { list });
const items = useWatch(todos?.items) ?? [];
@@ -714,8 +716,11 @@ when its items change:
const [text, setText] = usePageState(`lists.draft.${list}`, "");
const add = (event: { preventDefault(): void }) => {
event.preventDefault();
- if (todos === undefined || text.trim() === "") return;
- void todos.add(text, context);
+ // An empty todo too: the list's store refuses it, and the click's trace shows where.
+ if (todos === undefined) return;
+ // The click is a trace of its own: a root span of the page, the call to the list's worker under it.
+ const trace = { name: "page.add", attributes: { chars: text.length } };
+ void host.root((ctx) => todos.add(text, ctx), context, { trace }).catch(() => {});
setText("");
};
returnh(
@@ -762,7 +767,8 @@ when its items change:
messages.ts, and the facet contributes its catalogs to the page's Translations). useHold holds a reference while
the component is mounted and releases it on unmount; it is undefined on the first render. useWatch follows a member. usePageState is useState whose value the page keeps under a key instead of the
component: chapter IX shows why the half-typed todo needs it. Clicks call methods with BACKGROUND_CONTEXT, which never ends, so a click's call
-is not cancelled when the handler returns. todos?.variant === "owner" narrows the reference: only the owner's has
+is not cancelled when the handler returns; adding a todo runs in host.root (useHost gives the page's host), which
+makes the click a trace of its own (chapter XI). todos?.variant === "owner" narrows the reference: only the owner's has
clear, and TypeScript knows it. The panel's look reads the active theme's CSS variables, with class names prefixed
by the plugin's name (a *.module.css file would scope them instead):
@@ -1280,7 +1286,7 @@ replacing a list is a delete, then an import (decision "App kit naming and
an index of the app's, if it had one, is the app's.
-
XI
Seeing What Happened
One telemetry store for every process, one trace from a click to a commit, a span of your own, and a small viewer for the owner.
+
XI
Seeing What Happened
One telemetry store on the server, one trace from a click in the browser to a commit on a worker, a span of your own, whom each span was for, and a small viewer for the owner.
Ada says that adding a todo from her phone feels slow some evenings. Her todo crosses three processes: her page, the
server's relay and the list's worker. A log line per process would say what each one did, but not which lines belong
to her click, or where the time went. Chord records telemetry through pi-telemetry: spans (a named, timed
@@ -1288,10 +1294,12 @@ piece of work with attributes), events on spans, and me
adapter, given to createHost({ telemetry }) and NOOP by default, which decides where they go. A carrier in
each call's header names the caller's span, so the spans of one request, recorded by different processes, form one
trace (design §29). Mara wants to see them without reading SQL: a small trace viewer would be nice. This chapter
-records, follows one click, adds a span, reads the store from a script, and ends with the viewer.
-
Recording
+stores telemetry in one place, follows a click from the browser to the worker, adds a span, names whom each span was
+for, reads the store from a script, and ends with the viewer.
+
One store, on the server
The app keeps its telemetry in one SQLite file in the data directory, through Chord's store
-(@earendil-works/chord/telemetry-sqlite). The app's base code opens it, for any process that asks:
+(@earendil-works/chord/telemetry-sqlite), and the server is the only process that writes it. The app's base code opens
+it for the server:
/** The telemetry store's file in the data directory `dataDir`. */exportfunctiontelemetryFile(dataDir: string): string {
@@ -1313,77 +1321,94 @@ records, follows one click, adds a span, reads the store from a script, and ends
};
}
createSqliteTelemetry returns a recorder: an adapter that keeps each settled span in memory and writes them in
-one batch a second, one row per span with its events and links inline; metrics are summed per series and 10 seconds;
-rows older than seven days are deleted in the same batch, and when 10,000 spans wait, new ones are dropped and counted.
-Recording never runs SQL. The same object is the store's read interface, TelemetryStore (below).
-
Step 1 opened the store before the server (chapter I), and startTodo gives it to startServer, which passes it to the
-server's host and its HTTP component:
+one batch a second, one row per span with its events and links inline and its start and end in microseconds (a
+monotonic clock anchored to the wall clock, so spans of one process keep their order and spacing); metrics are summed
+per series and 10 seconds; rows older than seven days are deleted in the same batch, and when 10,000 spans wait, new
+ones are dropped and counted. Recording never runs SQL. The same object is the store's read interface,
+TelemetryStore (below).
+
Step 1 opened the store before the server (chapter I), and startTodo gives it to startServer:
-
// The server's spans and metrics; each list's worker opens the same file (`worker.ts`).
+
// The store the server writes: its own spans and metrics, and what the workers and recording pages send.
telemetry: telemetry.store,
-
The kit carries no spans between processes: each process records to the adapter it was given (slice 9, §5). So each
-list's worker opens the same file with a recorder of its own; the server names the file in the worker's environment:
-
-
env: { TODO_TELEMETRY: telemetryFile(dataDir) },
+
With a store, the kit does the rest. Every list's worker records in memory and sends its finished spans and metrics to
+the server once a second, over the link it already has; nothing tells a worker where the file is, and the worker's
+entry has no telemetry of its own to set up:
One file for every process is the app's choice. SQLite in WAL mode lets several processes write it, each waiting for
-the others' locks (up to 5 seconds), and every span id is global, so rows of different processes never collide. An app
-with a collector would give each process an adapter that exports there instead; nothing else changes.
-
One trace across three hosts
-
A page in a browser cannot record yet: startPage takes no telemetry adapter. Step 11 makes the page itself, as Ada's
-client was made in chapter VII: a client host of this process with a recorder of its own on the same file, signed in
-with the owner's cookie from step 1. A click is one span of the page, and the call runs under it:
startSpan runs the callback in a new root span; withTelemetry(span, context) makes it the context's span, so the
-call of add starts its chord.call under it and puts that span's carrier in the call's header. Each recorder writes
-once a second, so the step reads the store until the worker's spans are there, takes the older of the two traces whose
-first span is page.add and prints it with pi-telemetry's renderTrace (main.ts masks the lengths of time, which
-differ at every run):
Three processes recorded these six spans. The page recorded page.add and chord.call, the consumer's call until its
-reply. The server recorded chord.relay: it forwarded the call to the list's worker without knowing member names
-(#0 is the method's index). The worker recorded the rest: chord.method for the owner's Todos view, then the
-view's call of the store, which is a method span too, by worker, because the store acts as its host (chapter II).
-The commit is an event on the span its transaction ran under, and the renderer prints a span's events before its
-children. Its counts are documents and their ops; a commit of table rows counts neither, so this one says 0 and 0.
-Figure 5. One click, one trace: each process starts its spans with its own recorder, and the carrier in each call's header names the parent span on the other side.
+
A page records when the app asks it to, in its web entry:
+
+
// The page records its clicks and the calls under them, and sends them to the server every few seconds.
+ telemetry: true,
+
The page keeps its finished spans in memory and sends them every three seconds, and at once when it is hidden (a tab
+switched away may not come back). Both send to an object the kit serves on the server, WorkerTelemetry and
+PageTelemetry, whose ingest(batch) hands the spans to the store; its calls record no span of their own, or every
+batch would write a span about itself. A worker is the server's own child: its batches are written as they are,
+marked with its host. A page is anyone's: sign-in and the link's limits apply as for any call, and the server checks
+everything (next section).
+
One trace from the browser
+
The lists plugin's panel starts a trace where Ada's click starts: in its form's submit, a root span of the page's host
+with the call under it:
+
+
// The click is a trace of its own: a root span of the page, the call to the list's worker under it.
+ const trace = { name: "page.add", attributes: { chars: text.length } };
+ void host.root((ctx) => todos.add(text, ctx), context, { trace }).catch(() => {});
+
host.root(fn, context, { trace }) runs fn in a new root span of the host's adapter, ctx.telemetry that span, so
+the call of add starts its chord.call under it and puts that span's carrier in the call's header (useHost gives a
+component its page's host). Without a recording adapter it starts nothing. screenshots.ts signs the owner in to the
+page in Chromium, types honey and submits, then submits again with nothing typed; it reads the store until the
+worker's spans of both clicks are there and prints honey's trace with pi-telemetry's renderTrace (ids and lengths of
+time masked, which differ at every run):
Three processes recorded these six spans, and the server wrote them all. Each span says which host recorded it:
+chord.host is the host's address key (server, list|groceries), and browser for what a page sent. The page
+recorded page.add and chord.call, the consumer's call until its reply. The server recorded chord.relay: it
+forwarded the call to the list's worker without knowing member names (#0 is the method's index). The worker recorded
+the rest: chord.method for the owner's Todos view, then the view's call of the store, which is a method span too,
+by worker, because the store acts as its host (chapter II). The commit is an event on the span its transaction ran
+under, and the renderer prints a span's events before its children. It counts the documents it changed, their ops and
+the table rows its writes changed: the store keeps todos in a table, so one row and no documents. chord.principal
+says whom a span was for (below).
+Figure 5. One click, one trace: each process starts its spans with its own adapter, the carrier in each call's header names the parent span on the other side, and the page and the worker send their spans to the server, which writes them.
The carrier is a field of the call and hold headers, at most 8 keys of at most 256 characters each. The receiver
reads it with its own adapter; an absent or malformed one gives a new root, never an error. With the NOOP adapter
nothing is injected and no span starts: a span site costs one check (§30).
+
What a page sends, the server treats as untrusted. A batch is checked by its schema: at most 100 spans, every field
+bounded (names 128 characters, 32 attributes of at most 256 characters, 16 events, 8 links, a 500-character error
+message), ids printable ASCII and no control characters anywhere; a batch outside it is refused whole. A page's span
+ids are browser:<its host's incarnation>:<n>, and the server takes only spans whose id, trace and parent carry the
+calling page's own prefix: a page can neither write into another sender's trace nor pose as a server span, and the
+store never replaces a stored span. A page may send 1,200 spans a minute; past that they are dropped and counted in
+ingest's answer. The server marks each span chord.host: browser and stamps it with the principal of the link it came
+over: what a page says about itself counts for nothing.
The trace holds the click and nothing else. Work that serves many requests is its own trace, linked to the span that
first needed it: a worker's start (chord.resolve), an object's activation (chord.activate). A link is a reference to
a span of another trace; the renderer prints it and never follows it.
Trap: a trace starts where a span starts
-
The page took its reference to Todos before the click, under BACKGROUND_CONTEXT, and that hold is in no trace of
-the page: a context without a recording span records nothing. A host gives such work no span of its own, so its
-calls start no chord.call and send no carrier, and the provider's chord.method becomes a root of its own trace
+
The panel took its reference to Todos before the click (useHold), under BACKGROUND_CONTEXT, and that hold is in no
+trace of the page: a context without a recording span records nothing. A host gives such work no span of its own,
+so its calls start no chord.call and send no carrier, and the provider's chord.method becomes a root of its own trace
(the server's relay of the page's hold is one: chord.relay with member="hold"). So were the server's calls in step 2:
-on the worker each is a root chord.method. Start a span where the work begins, a click handler or a job, and pass it
-down with withTelemetry. The kit's HTTP component does this for every request: a chord.http span (method, route
-pattern, status), with the route's ctx under it.
+on the worker each is a root chord.method. Start a span where the work begins, a click handler or a job
+(host.root(fn, context, { trace })), and pass its ctx down. The kit's HTTP component does this for every request: a
+chord.http span (method, route pattern, status), with the route's ctx under it. Step 11 adds honey from the command
+line, as step 3 did; the request is the trace's root, the server's call to the worker under it:
+
+
The store's add checks the text in a span of its own:
@@ -1392,22 +1417,53 @@ pattern, status), with the route's ctx under it.
if (text === "") throwinvalid("args", "a todo needs text");
});
ctx.trace(name, attributes, fn) starts a child of the method's span and gives fn a child ctx with the same
-principal and get, so calls made inside are parented under it. It is the last line of the trace above:
-todos.validate chars=5. Its attributes are small, flat values; the todo's text is data and stays out of telemetry,
-its length is enough. When fn throws, the span fails and the error passes through as it was: the caller still gets
-invalid/args. A check this small does not need a span of its own; a lookup, a model request or a file read does.
+principal and get, so calls made inside are parented under it, and names its host as Chord's spans do. It is the last
+line of both traces above: todos.validate chord.host="list|groceries" chars=5. Its attributes are small, flat values;
+the todo's text is data and stays out of telemetry, its length is enough. When fn throws, the span fails and the error
+passes through as it was: the caller still gets invalid/args. A check this small does not need a span of its own; a
+lookup, a model request or a file read does.
+
Whom a span was for
+
Method and call spans record whom they were for: by default chord.principal, the principal's key (its canonical
+JSON). An app may record other attributes instead, through a hook each host takes (principalAttributes): the todo
+app's records a short key and, for a user, the name she signed in with:
Chord calls the hook for every span it records with a principal, so it must be synchronous and cheap; any caching is
+the hook's own (here a WeakMap by principal: a link stamps one principal object on all its calls). startTodo gives
+it to the server (principalAttributes beside telemetry) and the worker's entry to startWorker (above): each host
+has its own, the same function. The server also stamps a page's spans with it, for the principal of the link the batch
+came over: page.add above was the owner's. Only what the hook returns is recorded, never other app data, and the
+attributes are for the owner's eyes: the store, like the viewer below, is the owner's.
Reading
The store is read through TelemetryStore, pi-telemetry's read interface modeled on Jaeger's: findTraces (complete
traces, newest first, filtered by their first span's name, attributes, errors or a time window, paged by a cursor),
getTrace(id) and findMetrics (a metric's rows, one per series and 10 seconds, with count, sum, min and max).
-trace.ts is a script for a data directory, the app running or not:
+trace.ts is a script for a data directory, the app running or not. It opens the file with Chord's read-only store,
+openTelemetryStore (@earendil-works/chord/telemetry-sqlite/node), which implements only the read interface and
+opens SQLite read-only: it never writes the file the server writes, never creates or migrates it, and refuses a file
+without a store or with a store of another version.
-
if (id !== undefined) {
- const trace = await telemetry.store.getTrace(id);
+
// The shared file, read-only: the app's processes may be writing it meanwhile.
+const store = awaitopenTelemetryStore(telemetryFile(dataDir));
+try {
+ if (id !== undefined) {
+ const trace = await store.getTrace(id);
console.log(trace === undefined ? `no trace ${id}` : renderTrace(trace.spans));
} else {
- // Complete traces, newest first; a trace's first span (its root, by start time) names it.
- const { traces } = await telemetry.store.findTraces({ name: values.name, limit: 10 });
+ // Complete traces, newest first; a trace's first span (its root, by start time) names it, and the app's
+ // `principalAttributes` hook named whom each span was for.
+ const attributes = values.principal === undefined ? undefined : { "chord.principal": values.principal };
+ const { traces } = await store.findTraces({ name: values.name, attributes, limit: 10 });
for (const { traceId, spans } of traces) {
const start = Math.min(...spans.map((span) => span.startedMs));
const length = Math.max(...spans.map((span) => span.endedMs)) - start;
@@ -1416,110 +1472,122 @@ traces, newest first, filtered by their first span's name, attributes, error
}
// Metrics are stored per series and 10 s: a series is one set of attributes, here the direction.for (const direction of ["sent", "received"]) {
- const rows = await telemetry.store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
+ const rows = await store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
const bytes = rows.reduce((sum, row) => sum + row.sum, 0);
const messages = rows.reduce((sum, row) => sum + row.count, 0);
console.log(`chord.link.bytes ${direction}: ${bytes} bytes in ${messages} messages`);
}
}
-
Step 11 runs it twice from a process of its own: the newest page.add traces, then the refused one by its id (ids,
-lengths and counts masked):
-
-
trace.ts <data directory> --name page.add:
+
screenshots.ts runs it twice from a process of its own while the app runs: the newest page.add traces with a span
+for the owner, then the refused click by its id (ids, lengths and counts masked):
+
+
trace.ts <data directory> --name page.add --principal owner:
<id> page.add 6 spans <ms> error
<id> page.add 6 spans <ms>
chord.link.bytes sent: <n> bytes in <n> messages
chord.link.bytes received: <n> bytes in <n> messages
- trace.ts <data directory> <the refused one's id>:
- page.add error ChordError: a todo needs text
- chord.call type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="error" error call_failed: a todo needs text
- chord.relay type="todo.todos" member="#0" upstream.kind="list" outcome="error" error relay_failed: a todo needs text
- chord.method type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" outcome="error" error method_failed: a todo needs text
- chord.method type="todo.store" member="add" object="todo.store" principal.kind="worker" outcome="error" error method_failed: a todo needs text
- todos.validate chars=0 error trace_failed: a todo needs text
+trace.ts <data directory> <the refused one's id>:
+ page.add chord.host="browser" chars=0 chord.principal="owner" error trace_failed: a todo needs text
+ chord.call chord.host="browser" type="todo.todos" member="add" object="todo.todos(groceries)" address.kind="list" outcome="error" chord.principal="owner" error call_failed: a todo needs text
+ chord.relay chord.host="server" type="todo.todos" member="#0" upstream.kind="list" outcome="error" error relay_failed: a todo needs text
+ chord.method chord.host="list|groceries" type="todo.todos" member="add" object="todo.todos(groceries)" principal.kind="owner" chord.principal="owner" outcome="error" error method_failed: a todo needs text
+ chord.method chord.host="list|groceries" type="todo.store" member="add" object="todo.store" principal.kind="worker" chord.principal="worker:list|groceries" outcome="error" error method_failed: a todo needs text
+ todos.validate chord.host="list|groceries" chars=0 error trace_failed: a todo needs text
A trace's id is its root span's. The refused add failed on every span of its path, each with a fixed category
-(call_failed, relay_failed, method_failed, trace_failed) and the message cut at 500 characters; a cancelled call
-is not an error. It has no commit: nothing was written. chord.link.bytes counts the encoded messages of every host's
-links, each message once where it was sent and once where it was received. A metric carries no ids, only attributes
-with few values (direction), so its series stay few. A running recorder writes a metric's current 10 seconds only
-once they are over, or when it closes: trace.ts sees the page's bytes, whose recorder closed, and the server's and
-workers' of the intervals already over. The script is a recorder too, which records nothing; Chord ships only the
-store, so reads go through a recorder of the file.
+(trace_failed, call_failed, relay_failed, method_failed) and the message cut at 500 characters; a cancelled call
+is not an error. It has no commit: nothing was written. chord.link.bytes counts the encoded messages of the server's
+and the workers' links (a page sends no metrics), each message once where it was sent and once where it was received.
+A metric carries no ids, only attributes with few values (direction), so its series stay few. A metric's current 10
+seconds are sent and written only once they are over, or when a process closes.
A trace viewer
Two halves make the viewer: routes on the server that read the store, and a page that shows what they answer. Server
code of an app is either the start script or a plugin, and routes come from plugins (chapter III), so the routes are a
-small plugin, plugins/traces/. Its server facet serves one host-lifetime object, which opens the store's file as a
-native and contributes the routes while it is live:
+small plugin, plugins/traces/. Its server facet serves one host-lifetime object, which opens the store's file
+read-only as a native and contributes the routes while it is live:
host.serve(Traces, (_params, self) => {
const info = self.get(AppInfo, {});
- // The server records to the same file with its own recorder; this one only reads.
+ // The server records to the same file with its own recorder; this object only reads it.const telemetry = self.open(
- async () => openTelemetry(telemetryFile(info.dataDir.value)),
- (opened) => opened.close(),
+ async () => openTelemetryStore(telemetryFile(info.dataDir.value)),
+ (store) => store.close(),
);
self.contribute(Http, {
routes: {
- "GET /api/traces": async (_request, ctx) => {
+ "GET /api/traces": async (request, ctx) => {
ownerOnly(ctx);
- const { traces } = await telemetry.value.store.findTraces({ limit: RECENT });
+ // `?principal=user:u1`: the traces with a span for that principal (`principalAttributes`).
+ const principal = request.query.get("principal");
+ const attributes = principal === null ? undefined : { "chord.principal": principal };
+ const { traces } = await telemetry.value.findTraces({ limit: RECENT, attributes });
return { kind: "json", value: traces.map(({ traceId, spans }) => summaryOf(traceId, spans)) };
},
"GET /api/traces/:id": async (request, ctx) => {
ownerOnly(ctx);
- const trace = await telemetry.value.store.getTrace(request.params.id!);
+ const trace = await telemetry.value.getTrace(request.params.id!);
if (trace === undefined) return { kind: "error", status: 404 };
- return { kind: "json", value: { id: trace.traceId, spans: rowsOf(trace.spans) } };
+ return { kind: "json", value: { id: trace.traceId, spans: rowsOf(clamped(trace.spans)) } };
},
},
});
return {};
});
The kit gives a plugin the server's process values as objects: AppInfo names the data directory, and the API's server
-module re-exports it with the app's openTelemetry. self.open declares a native, opened once AppInfo is live and
-closed when the object ends; a reload of the plugin restarts the object and opens the file anew. An object may
-contribute when it lives as long as its host (local, empty params; decision R21). Spans name who called what, so
-only the owner reads them:
+module re-exports it with telemetryFile and Chord's openTelemetryStore. self.open declares a native, opened once
+AppInfo is live and closed when the object ends; a reload of the plugin restarts the object and opens the file anew.
+An object may contribute when it lives as long as its host (local, empty params; decision R21). Spans name who called
+what, so only the owner reads them:
functionownerOnly(ctx: Ctx): void {
if (ctx.principal?.kind !== "owner") throwforbidden("the traces are the owner's");
}
-
A route's principal is undefined only on a public route, which these are not; a signed-in member gets 403. rowsOf
-orders a trace's spans as a tree, parents before their children, and turns attributes, events and links into text. The
-spans name no host; the SQLite store's span ids begin with the id of the recorder, one per process, and the viewer
-calls each recorder a host:
A route's principal is undefined only on a public route, which these are not; a signed-in member gets 403. The list
+filters by whom the traces were for (?principal=), and shows each with its user's name or principal key. rowsOf
+orders a trace's spans as a tree, parents before their children, each row's host its span's chord.host, and turns
+attributes, events and links into text. A page's spans carry the page's clock, which may be off by seconds against the
+server's, so a span of the page could show starting after the server span under it. Before drawing, clamped moves
+the page's spans of the trace by the least amount that puts every server span under a page span inside it
+(plugins/traces/clock.ts, with its own test):
That is a detail of this store, not of the read interface (see the epilogue). The page is the app's own page at a
-second path: start.ts serves it at / and /traces (chapter I), and the web entry picks the viewer by the path:
+
The page is the app's own page at a second path: start.ts serves it at / and /traces (chapter I), and the web
+entry picks the viewer by the path:
// The page's one other path (`start.ts` serves the page at `/` and `/traces`).if (location.pathname === "/traces") returnh("main", { className: "wide" }, h("h1", null, "Todo"), h(TraceViewer));
TraceViewer (src/web/traces.ts) fetches the list and the trace chosen in the URL's fragment, and draws each span
-with its host, its length as a bar on the trace's time line, its attributes and events, a failed span in the theme's
-error colour, and each link as the ids it refers to, never fetched. The bars are positioned with inline styles, which
-the page's CSP allows only because React sets them through the DOM:
+with its host's name, its length as a bar on the trace's time line, its attributes and events, a failed span in the
+theme's error colour, and each link as the ids it refers to, never fetched. The bars are positioned with inline
+styles, which the page's CSP allows only because React sets them through the DOM:
wrote docs/tutorial/screens/traces.png
-the viewer shows page.add: 6 spans on 3 hosts
+the viewer shows page.add: 6 spans on 3 hosts
+wrote docs/tutorial/screens/traces-ada.png
+the viewer for user:u1 lists chord.method for ada
-Figure 6. The owner's trace viewer at /traces, in the Harbor theme: the newest traces (the refused add marked in red, the rest the app's own work: relays, activations, HTTP requests), and honey's trace as a tree across the page, the server and the worker. Starts are stored in whole milliseconds, so the bars of spans this short shift by up to one.
+Figure 6. The owner's trace viewer at /traces, in the Harbor theme: the newest traces (the refused click marked in red, each with whom it was for; the rest the app's own work: relays, activations, HTTP requests), and honey's trace as a tree across the page (browser), the server and the worker (list|groceries), all written by the server. Starts are stored to the microsecond, so even bars of a fraction of a millisecond line up.
+
+Figure 7. The traces for Ada (user:u1): the coffee screenshots.ts adds at its start as her (a root of the server acting as her), recorded by the list's worker; the app's hook named her ada.
The list is mostly the app's own work: each HTTP request, relayed hold and activation is a trace of its own, as the
-trap above says. A real viewer would filter by name (findTraces({ name })) or by error (errors: true), both of
-which the read interface has; this one is about 270 lines of TypeScript in two files and 20 of CSS, and no Chord API was added for it.
+trap above says. A real viewer would filter by name too (findTraces({ name })) or by error (errors: true), both of
+which the read interface has; this one is about 330 lines of TypeScript in three files and 20 of CSS, on Chord's
+read-only store.
XII
Testing One Plugin
testPlugin builds a plugin as production does and loads it alone on a host of its own.
@@ -1735,14 +1803,15 @@ chapter IX), a plugin's contract.ts could not be imported outsi
chapter I), plugins could not throw Chord's invalid (Chord exports invalid and forbidden, chapter II), and a
live window over a table needed a one-time read in self.open, which made the store restart at every reload of its
plugin (now self.query, a live query Chord runs, chapters II and IX).
-
Found while writing the telemetry chapter, still open. A page cannot record: startPage takes no telemetry
-adapter, so chapter XI's page is a client host in Node, and a browser's click starts its trace at the server's relay.
-chord.commit counts documents and their ops only, so a commit that wrote table rows reports documents=0 ops=0.
-Spans name no host, and the viewer tells processes apart by the SQLite store's id format (<recorder>:<n>), which
-the read interface does not promise. The store keeps a span's start in whole milliseconds (Date.now()) and its
-length in fractions (performance.now()), so on a trace of a few milliseconds the viewer's bars shift by up to one. And
-reading needs a recorder: chord/telemetry-sqlite exports no read-only
-store, so a script or a viewer opens a recorder that records nothing.
+
Found while writing the telemetry chapter, and fixed in Chord since.chord.commit counted documents and their
+ops only, so a commit that wrote table rows reported documents=0 ops=0 (it counts rows now). Spans named no host,
+and the viewer told processes apart by the SQLite store's id format (every Chord span carries chord.host now). The
+store kept a span's start in whole milliseconds, so the viewer's bars shifted by up to one (starts are microseconds
+now, the store's version 2, which a recorder migrates its file to). Reading needed a recorder that recorded nothing
+(openTelemetryStore reads read-only). A page could not record, so the chapter's click was a client host in Node
+standing in for the page (startPage({ telemetry: true }) now, its spans sent to the server). And every process wrote
+the one file itself, each worker told its path (the server is the store's only writer now, the workers and pages
+sending it their spans), and spans said nothing of whom they were for (principalAttributes).
A
The App's Files
@@ -1757,7 +1826,7 @@ store, so a script or a viewer opens a recorder that records nothing.
src/principals.ts
base
-
the principal type
+
the principal type, principalAttributes (whom a span was for)
package.json
@@ -1772,12 +1841,12 @@ store, so a script or a viewer opens a recorder that records nothing.
src/api/v1/server.ts
base, server
-
Http, the ephemeral login, AppInfo, openTelemetry
+
Http, the ephemeral login, AppInfo, telemetryFile, openTelemetryStore