feat(chord): telemetry from pages and workers stored by the server, who a span was for

The four telemetry gaps (rows on commits, chord.host, microsecond times, a read-only store) and: pages record and
send span batches over their link to the server's ingest object (checked, capped, rate-limited, stamped browser and
with the link's principal); workers send theirs too, so the server is the store's only writer; principalAttributes
lets an app choose what a span records about its principal. The tutorial shows a real page's click traced to the
worker, with the trace viewer.
This commit is contained in:
Mario Zechner 2026-11-04 11:59:14 +00:00
parent 80f7a11494
commit ff33ef42c4
95 changed files with 3358 additions and 967 deletions

View file

@ -1,7 +1,7 @@
// The tutorial's app (`docs/tutorial/book.md`): a shared todo app built with the app kit, one worker per list. Each
// numbered step is the tutorial's chapter of that number: the server and the owner's sign-in, a list on its worker,
// routes, the page, the terminal, a theme plugin added while the app runs, a second user, languages, an edit applied in
// place, a list exported and imported, and a trace across the page, the server and a worker, read back from the app's
// place, a list exported and imported, and a request's trace across the server and a worker, read back from the app's
// telemetry store. The lists plugin's own test is `plugins/lists/worker.test.ts`; `screenshots.ts` shows the page and
// the trace viewer in Chromium; `trace.ts` reads a data directory's traces.
// Run from packages/chord (with Node 26.5.1 or later the terminal client runs too; `--serve` keeps the app running):
@ -16,7 +16,7 @@ import type { StoredTrace } from "@earendil-works/pi-telemetry";
import { renderTrace } from "@earendil-works/pi-telemetry/render";
import { List } from "@todo/app/v1";
import { WebSocket } from "ws";
import { BACKGROUND_CONTEXT as context, withTelemetry } from "../../../src/context/index.ts";
import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts";
import {
type ChordError,
createHost,
@ -249,37 +249,27 @@ await app.importData(archive, List({ id: "groceries" }), context).catch((error:
console.log(" importing over groceries:", error.message.replace(base, "…"));
});
// 11. Telemetry. Every process of the app records to the store in the data directory (step 1 opened it for the server,
// `worker.ts` opens it in each list's worker). A page stands in here as a client host of this process with a
// recorder of its own on the same file, signed in with the owner's cookie of step 1. A click is one span of the
// page; the call under it goes through the server's relay into the list's worker, and the carriers on the wire make
// one trace of the three processes' spans. The store's `add` checks the text in a span of its own (`ctx.trace`).
const pageTelemetry = await openTelemetry(telemetryFile(dataDir));
const page = createHost<TodoPrincipal>({
telemetry: pageTelemetry.store,
onError: (error) => console.error("page:", error.kind),
});
await page.load([], context);
const pageSocket = new WebSocket(`${url.replace(/^http/, "ws")}/chord`, { headers: { cookie } });
page.connect(webSocketLink(pageSocket as unknown as WebSocketLike), { principal: { kind: "server" } }, context);
const pageTodos = await page.get(Todos, { list: "groceries" }, context);
const click = (text: string) =>
pageTelemetry.store.startSpan({ name: "page.add" }, (span) => pageTodos.add(text, withTelemetry(span, context)));
await click("honey");
await click(" ").catch(() => "refused");
pageTodos.release();
await page.close(context);
await pageTelemetry.close();
// Each recorder writes what settled once a second: read until the worker's spans of both clicks are there.
const [refusedTrace, honeyTrace] = await written(telemetry.store, "page.add", 2, "todos.validate");
console.log("11. adding honey from the page:");
// 11. Telemetry. The server records to the store in the data directory (step 1 opened it) and writes what every list's
// worker sends it. A command-line client adds honey and a blank todo: each request is a `chord.http` trace of the
// server, the call into the list's worker under it, and the store's `add` checks the text in a span of its own
// (`ctx.trace`). A page's click, recorded by the page itself, is `screenshots.ts`'s (it needs Chromium).
for (const text of ["honey", " "]) {
await fetch(`${url}/api/lists/groceries`, {
method: "POST",
headers: { ...cli, "content-type": "application/json" },
body: JSON.stringify(text),
});
}
// The worker sends every second, the server writes every second: read until the worker's spans of both are there.
const [refusedTrace, honeyTrace] = await written(telemetry.store, "chord.http", 2, "todos.validate");
console.log("11. adding honey from the command line:");
printTelemetry(renderTrace(honeyTrace!.spans));
// `trace.ts` reads the same file from a process of its own: the newest traces, then one by its id.
const traceScript = fileURLToPath(new URL("./trace.ts", import.meta.url));
const read = async (...args: string[]) =>
(await promisify(execFile)(process.execPath, [...process.execArgv, traceScript, ...args])).stdout;
console.log(" trace.ts <data directory> --name page.add:");
printTelemetry(await read(dataDir, "--name", "page.add"));
console.log(" trace.ts <data directory> --name chord.http --principal owner:");
printTelemetry(await read(dataDir, "--name", "chord.http", "--principal", "owner"));
console.log(" trace.ts <data directory> <the refused one's id>:");
printTelemetry(await read(dataDir, refusedTrace!.traceId));

View file

@ -4,7 +4,7 @@
// texts are in the page's language (`useMessages`), and its catalogs are its contribution to the page's `Translations`.
import { BACKGROUND_CONTEXT as context, defineFacet, Translations } from "@todo/app/v1";
import { Panels, useHold, useMessages, usePageState, useWatch } from "@todo/app/v1/web";
import { Panels, useHold, useHost, useMessages, usePageState, useWatch } from "@todo/app/v1/web";
import { createElement as h } from "react";
import { Todos } from "./contract.ts";
import { messages } from "./messages.ts";
@ -12,6 +12,7 @@ import "./lists.css";
function TodoList() {
const { t } = useMessages(messages);
const host = useHost();
const list = location.hash.slice(1) || "groceries";
const todos = useHold(Todos, { list });
const items = useWatch(todos?.items) ?? [];
@ -20,8 +21,11 @@ function TodoList() {
const [text, setText] = usePageState(`lists.draft.${list}`, "");
const add = (event: { preventDefault(): void }) => {
event.preventDefault();
if (todos === undefined || text.trim() === "") return;
void todos.add(text, context);
// An empty todo too: the list's store refuses it, and the click's trace shows where.
if (todos === undefined) return;
// The click is a trace of its own: a root span of the page, the call to the list's worker under it.
const trace = { name: "page.add", attributes: { chars: text.length } };
void host.root((ctx) => todos.add(text, ctx), context, { trace }).catch(() => {});
setText("");
};
return h(

View file

@ -0,0 +1,60 @@
// The viewer's clock correction: a page's spans moved by the least amount that puts the server's spans under them
// inside them; nothing moved when they fit, or when no server span is under a page span.
// Run from packages/chord:
// node ../../node_modules/vitest/dist/cli.js --run test/examples/61-tutorial/plugins/traces/clock.test.ts
import type { StoredSpan } from "@todo/app/v1/server";
import { describe, expect, it } from "vitest";
import { clamped } from "./clock.ts";
/** A span `id` on `host` from `startedMs` to `endedMs`, under `parentId`. */
function span(id: string, host: string, startedMs: number, endedMs: number, parentId?: string): StoredSpan {
return {
id,
traceId: "t",
...(parentId === undefined ? {} : { parentId }),
name: id,
startedMs,
endedMs,
attributes: { "chord.host": host },
events: [],
links: [],
status: { status: "ok" },
};
}
/** Each span's times after the correction. */
function times(spans: readonly StoredSpan[]): [string, number, number][] {
return clamped(spans).map(({ id, startedMs, endedMs }) => [id, startedMs, endedMs]);
}
describe("clamped", () => {
it("moves a page that runs late back, so its call contains the server's relay", () => {
const spans = [span("click", "browser", 5000, 5010), span("call", "browser", 5001, 5009, "click")];
spans.push(span("relay", "server", 2002, 2008, "call"));
expect(times(spans)).toEqual([
["click", 2001, 2011],
["call", 2002, 2010],
["relay", 2002, 2008],
]);
});
it("moves a page that runs early forward by the least amount", () => {
const spans = [span("call", "browser", 100, 110), span("relay", "server", 1101, 1105, "call")];
expect(times(spans)).toEqual([
["call", 1095, 1105],
["relay", 1101, 1105],
]);
});
it("moves nothing when the server's spans fit, or when no server span is under a page span", () => {
const fitting = [span("call", "browser", 100, 110), span("relay", "server", 101, 109, "call")];
expect(clamped(fitting)).toBe(fitting);
const alone = [span("click", "browser", 5000, 5010), span("other", "server", 1, 2)];
expect(clamped(alone)).toBe(alone);
});
it("centres a server child longer than its page parent", () => {
const spans = [span("call", "browser", 100, 102), span("relay", "server", 200, 210, "call")];
expect(times(spans)[0]).toEqual(["call", 204, 206]);
});
});

View file

@ -0,0 +1,42 @@
// The trace viewer's clocks: a page's spans come with the page's clock, which may be off by seconds against the
// server's; a trace's page spans are moved onto the server's clock by its own spans under them.
import type { StoredSpan } from "@todo/app/v1/server";
/**
* `spans` with the page's moved onto the server's clock: a page's clock may be off by seconds, so its spans are moved
* by the least amount that puts each server span under a page span inside it (`skewOf`); none without such a pair.
*/
export function clamped(spans: readonly StoredSpan[]): readonly StoredSpan[] {
const skew = skewOf(spans);
if (skew === 0) return spans;
return spans.map((span) =>
hostOf(span) === "browser" ? { ...span, startedMs: span.startedMs + skew, endedMs: span.endedMs + skew } : span,
);
}
/**
* The shift of the page's spans closest to none that puts every server child of a page span inside it: at least each
* child's end past its parent's, at most each child's start past its parent's (centred when no shift fits both: a
* child longer than its parent).
*/
function skewOf(spans: readonly StoredSpan[]): number {
const byId = new Map(spans.map((span) => [span.id, span]));
let least = Number.NEGATIVE_INFINITY;
let most = Number.POSITIVE_INFINITY;
for (const child of spans) {
const parent = child.parentId === undefined ? undefined : byId.get(child.parentId);
if (parent === undefined || hostOf(parent) !== "browser" || hostOf(child) === "browser") continue;
least = Math.max(least, child.endedMs - parent.endedMs);
most = Math.min(most, child.startedMs - parent.startedMs);
}
if (least === Number.NEGATIVE_INFINITY) return 0;
if (least > most) return (least + most) / 2;
return Math.min(Math.max(0, least), most);
}
/** The host that recorded `span`, as every span of Chord's names it (`chord.host`). */
export function hostOf(span: StoredSpan): string | undefined {
const host = span.attributes["chord.host"];
return typeof host === "string" ? host : undefined;
}

View file

@ -1,10 +1,12 @@
// The trace viewer's server half: the owner's JSON routes over the app's telemetry store, which the page's viewer at
// `/traces` reads (`src/web/traces.ts`). One object per server opens the store's file in the data directory (a native)
// and contributes the routes while it is live; they read through the store's read interface only (`findTraces`,
// `getTrace`) and never follow a link to another trace.
// `/traces` reads (`src/web/traces.ts`). One object per server opens the store's file in the data directory read-only
// (a native) and contributes the routes while it is live; they read through the store's read interface only
// (`findTraces`, filtered by who the spans were for, and `getTrace`) and never follow a link to another trace. A page's
// spans come with the page's clock: the trace moves them onto the server's (`clamped`).
import { type Ctx, defineFacet, forbidden, local } from "@todo/app/v1";
import { AppInfo, Http, openTelemetry, type StoredSpan, telemetryFile } from "@todo/app/v1/server";
import { AppInfo, Http, openTelemetryStore, type StoredSpan, telemetryFile } from "@todo/app/v1/server";
import { clamped, hostOf } from "./clock.ts";
/** The viewer's reader, once per server: empty params, so it lives as long as the server and may contribute. */
const Traces = local("todo.traces", { version: 1, params: {}, members: {} });
@ -15,23 +17,26 @@ const RECENT = 40;
export default defineFacet((host) => {
host.serve(Traces, (_params, self) => {
const info = self.get(AppInfo, {});
// The server records to the same file with its own recorder; this one only reads.
// The server records to the same file with its own recorder; this object only reads it.
const telemetry = self.open(
async () => openTelemetry(telemetryFile(info.dataDir.value)),
(opened) => opened.close(),
async () => openTelemetryStore(telemetryFile(info.dataDir.value)),
(store) => store.close(),
);
self.contribute(Http, {
routes: {
"GET /api/traces": async (_request, ctx) => {
"GET /api/traces": async (request, ctx) => {
ownerOnly(ctx);
const { traces } = await telemetry.value.store.findTraces({ limit: RECENT });
// `?principal=user:u1`: the traces with a span for that principal (`principalAttributes`).
const principal = request.query.get("principal");
const attributes = principal === null ? undefined : { "chord.principal": principal };
const { traces } = await telemetry.value.findTraces({ limit: RECENT, attributes });
return { kind: "json", value: traces.map(({ traceId, spans }) => summaryOf(traceId, spans)) };
},
"GET /api/traces/:id": async (request, ctx) => {
ownerOnly(ctx);
const trace = await telemetry.value.store.getTrace(request.params.id!);
const trace = await telemetry.value.getTrace(request.params.id!);
if (trace === undefined) return { kind: "error", status: 404 };
return { kind: "json", value: { id: trace.traceId, spans: rowsOf(trace.spans) } };
return { kind: "json", value: { id: trace.traceId, spans: rowsOf(clamped(trace.spans)) } };
},
},
});
@ -47,17 +52,23 @@ function ownerOnly(ctx: Ctx): void {
/** The attributes of a trace's first span the list shows: an object's type and member, an HTTP route. */
const DETAIL = ["type", "member", "route"];
/** A trace as the list shows it: its first span's name and details, its size, its processes, its length, its failure. */
function summaryOf(id: string, spans: readonly StoredSpan[]) {
/**
* A trace as the list shows it: its first span's name and details, whom it was for (a user's name, else the principal
* key of its first span that has one), its size, its hosts, its length, its failure.
*/
function summaryOf(id: string, stored: readonly StoredSpan[]) {
const spans = clamped(stored);
const start = Math.min(...spans.map((span) => span.startedMs));
// A stored trace has at least one span; its spans come in start order.
const first = spans[0]!;
const who = spans.find(({ attributes }) => attributes["chord.principal"] !== undefined)?.attributes;
return {
id,
name: first.name,
detail: DETAIL.flatMap((key) => first.attributes[key] ?? []).join(" "),
principal: String(who?.["user.name"] ?? who?.["chord.principal"] ?? ""),
spans: spans.length,
processes: new Set(spans.map(processOf)).size,
hosts: new Set(spans.flatMap((span) => hostOf(span) ?? [])).size,
durationMs: Math.max(...spans.map((span) => span.endedMs)) - start,
error: spans.some((span) => span.status.status === "error"),
};
@ -65,8 +76,9 @@ function summaryOf(id: string, spans: readonly StoredSpan[]) {
/**
* The trace's spans as the tree shows them, parents before their children (in start order), each with its depth, its
* process, its offset and length from the trace's start, and its attributes, events, links and error as text. A span
* whose parent is not in the trace (another trace's, or one not stored) is a root.
* host, its offset and length from the trace's start, and its attributes, events, links and error as text. A span
* whose parent is not in the trace (another trace's, or one not stored) is a root. A span naming no host (one an app
* started on an adapter itself) is shown on its parent's.
*/
function rowsOf(spans: readonly StoredSpan[]) {
const start = Math.min(...spans.map((span) => span.startedMs));
@ -80,24 +92,26 @@ function rowsOf(spans: readonly StoredSpan[]) {
else siblings.push(span);
}
const rows: ReturnType<typeof rowOf>[] = [];
const visit = (span: StoredSpan, depth: number): void => {
rows.push(rowOf(span, depth, start));
for (const child of children.get(span.id) ?? []) visit(child, depth + 1);
const visit = (span: StoredSpan, depth: number, around: string): void => {
const host = hostOf(span) ?? around;
rows.push(rowOf(span, depth, host, start));
for (const child of children.get(span.id) ?? []) visit(child, depth + 1, host);
};
for (const root of children.get(undefined) ?? []) visit(root, 0);
for (const root of children.get(undefined) ?? []) visit(root, 0, "?");
return rows;
}
/** One span of the tree (see `rowsOf`). */
function rowOf(span: StoredSpan, depth: number, start: number) {
/** One span of the tree (see `rowsOf`), recorded on `host`. */
function rowOf(span: StoredSpan, depth: number, host: string, start: number) {
const { status } = span;
return {
name: span.name,
depth,
process: processOf(span),
host,
offsetMs: span.startedMs - start,
durationMs: span.endedMs - span.startedMs,
attributes: pairs(span.attributes),
// Its host is shown as its tag.
attributes: pairs(span.attributes).filter((pair) => !pair.startsWith("chord.host=")),
events: span.events.map((event) => [event.name, ...pairs(event.attributes)].join(" ")),
// A reference to a span of another trace, shown and never followed.
links: span.links.map((link) => `trace ${link.traceId} span ${link.spanId}`),
@ -105,11 +119,6 @@ function rowOf(span: StoredSpan, depth: number, start: number) {
};
}
/** The recording process of `span`: the SQLite store's span ids are `<recorder>:<n>`, one recorder per process. */
function processOf(span: StoredSpan): string {
return span.id.slice(0, span.id.lastIndexOf(":"));
}
/** Attributes as `key=value` texts, the values as JSON, fractions at two decimals (`duration=0.41`). */
function pairs(attributes: StoredSpan["attributes"]): string[] {
return Object.entries(attributes).map(([key, value]) => {

View file

@ -1,18 +1,22 @@
// Screenshots of the tutorial's page (`docs/tutorial/screens/`), taken in headless Chromium: the page after the owner
// signed in with the owner link, the page in the Harbor theme added while it runs, the page after an edit of the
// lists plugin's web facet applied in place, the page after the owner chose French, with the list's notice, and the
// owner's trace viewer showing a todo added from a page.
// page before and after the owner adds honey (the page records its clicks), the trace viewer showing that click's
// trace, then the traces for Ada.
// Run from packages/chord, with Chromium (`CHROMIUM_BIN`, else `chromium-docker` on the `PATH`):
// node --conditions=source --experimental-strip-types test/examples/61-tutorial/screenshots.ts docs/tutorial/screens
import { execFile } from "node:child_process";
import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { WebSocket } from "ws";
import { BACKGROUND_CONTEXT as context, withTelemetry } from "../../../src/context/index.ts";
import { createHost, LocaleChoice, type WebSocketLike, webSocketLink } from "../../../src/index.ts";
import { promisify } from "node:util";
import type { StoredTrace } from "@earendil-works/pi-telemetry";
import { renderTrace } from "@earendil-works/pi-telemetry/render";
import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts";
import { LocaleChoice } from "../../../src/index.ts";
import { Browser, CHROMIUM } from "../../node/web/browser.ts";
import { Todos } from "./plugins/lists/contract.ts";
import type { TodoPrincipal } from "./src/principals.ts";
import { openTelemetry, telemetryFile } from "./src/telemetry.ts";
import { startTodo, todoDirectories } from "./start.ts";
@ -115,36 +119,42 @@ try {
console.log(`after choosing French: lang="${lang}", ${JSON.stringify(french)} still typed`);
console.log("the page did not reload:", await browser.evaluate<boolean>("window.marked === true"));
// 5. Honey added from a page that records, and a blank todo refused (main.ts's step 11: a client host with a recorder
// of its own, here signed in with the owner token), then the owner opens the trace viewer on honey's trace.
const pageTelemetry = await openTelemetry(telemetryFile(dataDir));
const page = createHost<TodoPrincipal>({ telemetry: pageTelemetry.store, onError: () => {} });
await page.load([], context);
const token = (await readFile(app.ownerTokenFile, "utf8")).trim();
const origin = new URL(app.ownerLink()).origin;
const socket = new WebSocket(`${origin.replace(/^http/, "ws")}/chord`, {
headers: { authorization: `Bearer ${token}` },
});
page.connect(webSocketLink(socket as unknown as WebSocketLike), { principal: { kind: "server" } }, context);
const todos = await page.get(Todos, { list: "groceries" }, context);
const click = (text: string) =>
pageTelemetry.store.startSpan({ name: "page.add" }, (span) => todos.add(text, withTelemetry(span, context)));
await click("honey");
await click("").catch(() => "refused");
todos.release();
await page.close(context);
await pageTelemetry.close();
// Each recorder writes what settled once a second: read until the worker's span of honey's add is there.
const honey = async () => {
// 5. The page records (`startPage({ telemetry: true })`): the owner adds honey, then clicks again with nothing typed;
// each click is a trace of the page's, the call under it through the server's relay into the list's worker. The
// page sends its spans every few seconds, the worker every second, the server writes every second: read until the
// worker's span of each click is there, print honey's trace, read the store with `trace.ts`, then open the viewer.
await browser.evaluate("document.querySelector('.lists-panel input').select()");
await browser.type("honey");
await shoot("click-before.png");
const submit = "document.querySelector('.lists-panel form').requestSubmit()";
await browser.evaluate(submit);
await browser.wait(
"[...document.querySelectorAll('.lists-panel li')].some((item) => item.textContent.startsWith('honey'))",
);
await browser.evaluate("document.querySelector('.lists-panel input').blur()");
await shoot("click-after.png");
await browser.evaluate(submit);
const clicks = async () => {
await telemetry.store.flush();
const [, trace] = (await telemetry.store.findTraces({ name: "page.add", limit: 2 })).traces;
return trace?.spans.some(({ name }) => name === "todos.validate") ? trace : undefined;
const { traces } = await telemetry.store.findTraces({ name: "page.add", limit: 2 });
return traces.filter(({ spans }) => spans.some(({ name }) => name === "todos.validate"));
};
let trace = await honey();
while (trace === undefined) {
let traces = await clicks();
while (traces.length < 2) {
await new Promise((done) => setTimeout(done, 100));
trace = await honey();
traces = await clicks();
}
const [refused, trace] = traces as [StoredTrace, StoredTrace];
console.log("adding honey from the page:");
printTelemetry(renderTrace(trace.spans));
const traceScript = fileURLToPath(new URL("./trace.ts", import.meta.url));
const read = async (...args: string[]) =>
(await promisify(execFile)(process.execPath, [...process.execArgv, traceScript, ...args])).stdout;
console.log("trace.ts <data directory> --name page.add --principal owner:");
printTelemetry(await read(dataDir, "--name", "page.add", "--principal", "owner"));
console.log("trace.ts <data directory> <the refused one's id>:");
printTelemetry(await read(dataDir, refused.traceId));
const origin = new URL(app.ownerLink()).origin;
await browser.call("Emulation.setDeviceMetricsOverride", {
width: 900,
height: 640,
@ -158,9 +168,33 @@ try {
"new Set([...document.querySelectorAll('.traces-tree .traces-host')].map((tag) => tag.textContent)).size",
);
console.log(`the viewer shows page.add: 6 spans on ${hosts} hosts`);
// 6. The traces for Ada (`user:u1`, the app's `principalAttributes` names her): her todo of the start, a call of the
// server's root acting as her, recorded by the list's worker with her name.
await browser.evaluate("document.querySelector('.traces-filter').focus()");
await browser.type("user:u1");
await browser.wait("document.querySelector('.traces nav a small')?.textContent.includes('ada')");
await browser.evaluate("document.querySelector('.traces nav a').click()");
await browser.wait("document.querySelectorAll('.traces-tree li').length > 0");
await shoot("traces-ada.png");
const listed = await browser.evaluate<string[]>(
"[...document.querySelectorAll('.traces nav a')].map((a) => a.querySelector('span').textContent + ' for ' + a.querySelectorAll('small')[0].textContent.split(' · ').at(-1))",
);
console.log(`the viewer for user:u1 lists ${[...new Set(listed)].join(", ")}`);
} finally {
await browser.close();
await app.close(context);
await telemetry.close();
await rm(base, { recursive: true, force: true });
}
/** Prints telemetry indented, as the same text at every run: span ids, lengths of time and byte counts masked. */
function printTelemetry(text: string): void {
const steady = text
.replaceAll(/browser:[0-9a-f-]{36}:\d+|[0-9a-f-]{36}:\d+/g, "<id>")
.replaceAll(/ \(\+?[\d.]+ms\)/g, "")
.replaceAll(/duration=[\d.]+/g, "duration=<ms>")
.replaceAll(/ [\d.]+ ms/g, " <ms>")
.replaceAll(/\d+ bytes in \d+ messages/g, "<n> bytes in <n> messages");
for (const line of steady.trim().split("\n")) console.log(` ${line}`);
}

View file

@ -1,6 +1,6 @@
// The todo app's API v1 for server facets (`@todo/app/v1/server`): the HTTP registry plugins add routes to, the kit's
// pick-a-name login as a facet a one-line plugin exports, and the app's telemetry store (where it is, how a server
// facet opens it to read) with the kit's `AppInfo`, which names the data directory.
// pick-a-name login as a facet a one-line plugin exports, and the app's telemetry store (where it is, and Chord's
// read-only store a server facet opens it with) with the kit's `AppInfo`, which names the data directory.
import { ephemeralLoginFacet } from "../../../../../../src/node/app/index.ts";
import type { TodoPrincipal } from "../../principals.ts";
@ -8,7 +8,8 @@ import type { TodoPrincipal } from "../../principals.ts";
export type { StoredSpan, StoredTrace } from "@earendil-works/pi-telemetry";
export { AppInfo } from "../../../../../../src/node/app/index.ts";
export { Http } from "../../../../../../src/node/http/index.ts";
export { openTelemetry, type TodoTelemetry, telemetryFile } from "../../telemetry.ts";
export { openTelemetryStore } from "../../../../../../src/telemetry-sqlite/node.ts";
export { telemetryFile } from "../../telemetry.ts";
/** The kit's ephemeral login: `POST /_chord/login { name }` signs in as `{ kind: "user", id, name }`. */
export const login = ephemeralLoginFacet<TodoPrincipal>();

View file

@ -4,7 +4,7 @@
import type { ComponentType } from "react";
import { defineRegistry } from "../../../../../../src/index.ts";
export { useHold, useMessages, usePageState, useWatch } from "../../../../../../src/react/index.ts";
export { useHold, useHost, useMessages, usePageState, useWatch } from "../../../../../../src/react/index.ts";
/** A panel of the page: its name (a key, never shown) and its React component, which shows its own title. */
export type Panel = { readonly name: string; readonly Component: ComponentType };

View file

@ -6,3 +6,23 @@ export type TodoPrincipal =
| { readonly kind: "worker"; readonly list: string }
| { readonly kind: "owner" }
| { readonly kind: "user"; readonly id: string; readonly name: string };
/** The names the telemetry records, by principal: a principal crossing a link is one object per link, made once. */
const recorded = new WeakMap<TodoPrincipal, Readonly<Record<string, string>>>();
/**
* Who a span was for, as the app's telemetry records it (the hosts' `principalAttributes`): a short principal key, and
* a user's name, which the owner's trace viewer shows. Cheap and synchronous, as Chord calls it per recorded span; the
* cache is the hook's own.
*/
export function principalAttributes(principal: TodoPrincipal): Readonly<Record<string, string>> {
let attributes = recorded.get(principal);
if (attributes === undefined) {
attributes =
principal.kind === "user"
? { "chord.principal": `user:${principal.id}`, "user.name": principal.name }
: { "chord.principal": principal.kind === "worker" ? `worker:${principal.list}` : principal.kind };
recorded.set(principal, attributes);
}
return attributes;
}

View file

@ -1,6 +1,6 @@
// The todo app's telemetry store: one SQLite file in the data directory, `telemetry.sqlite`. The server and every
// list's worker record to it, each process with a recorder of its own on the shared file; readers (the trace viewer,
// `trace.ts`) open it the same way and read through the store's read interface.
// The todo app's telemetry store: one SQLite file in the data directory, `telemetry.sqlite`. The server is its only
// writer: its own spans, and those every list's worker and every recording page send it; readers (the trace viewer,
// `trace.ts`) open it read-only with Chord's `openTelemetryStore` and read through the store's read interface.
import { mkdir } from "node:fs/promises";
import { dirname, join } from "node:path";

View file

@ -33,5 +33,7 @@ void startPage({
connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`),
modules: { "@todo/app/v1": v1, "@todo/app/v1/web": v1web, react: React, "react/jsx-runtime": jsx },
core: [{ name: "@todo/web", facet: core }],
// The page records its clicks and the calls under them, and sends them to the server every few seconds.
telemetry: true,
render: (host) => createRoot(document.getElementById("root")!).render(h(HostProvider, { host }, h(Page))),
});

View file

@ -4,12 +4,13 @@ main { max-width: 26rem; margin: 1.5rem auto; padding: 0 1rem; }
h1 { color: var(--accent); font-size: 1.4rem; margin: 0 0 1rem; }
h2 { font-size: 0.8rem; text-transform: uppercase; letter-spacing: 0.08em; color: var(--muted); }
/* The trace viewer (`/traces`): the list beside the chosen trace's tree, a colour per process. */
/* The trace viewer (`/traces`): the list (filtered by principal) beside the chosen trace's tree, a colour per host. */
main.wide { max-width: 60rem; }
.traces { display: grid; grid-template-columns: 13rem 1fr; gap: 1rem; align-items: start; }
.traces nav a { display: block; padding: 0.3rem 0.4rem; color: var(--fg); text-decoration: none; border-left: 2px solid transparent; }
.traces nav a.traces-chosen { border-left-color: var(--accent); background: color-mix(in srgb, var(--fg) 8%, transparent); }
.traces nav small { display: block; color: var(--muted); font-size: 0.75rem; }
.traces-filter { width: 100%; box-sizing: border-box; margin-bottom: 0.5rem; background: transparent; color: var(--fg); border: 1px solid var(--muted); border-radius: 3px; padding: 0.2rem 0.4rem; }
.traces-tree { list-style: none; margin: 0; padding: 0; font-size: 0.8rem; }
.traces-tree li { padding: 0.25rem 0; border-bottom: 1px solid color-mix(in srgb, var(--muted) 30%, transparent); }
.traces-span { display: flex; gap: 0.5rem; align-items: baseline; }

View file

@ -1,7 +1,8 @@
// The owner's trace viewer, the page at `/traces`: the newest traces of the app's telemetry store and the one chosen
// (its id in the URL's fragment) as a tree: each span with its host, its length as a bar on the trace's time line, its
// attributes and events, its error marked, and its links as references to other traces, never followed. It reads the
// traces plugin's JSON routes (`plugins/traces/server.ts`), which answer the owner only.
// The owner's trace viewer, the page at `/traces`: the newest traces of the app's telemetry store (all, or those for
// one principal: `user:u1`, `owner`), each with whom it was for, and the one chosen
// (its id in the URL's fragment) as a tree: each span with its host (`chord.host`), its length as a bar on the trace's
// time line, its attributes and events, its error marked, and its links as references to other traces, never
// followed. It reads the traces plugin's JSON routes (`plugins/traces/server.ts`), which answer the owner only.
import { createElement as h, useEffect, useState } from "react";
@ -10,8 +11,9 @@ type Summary = {
readonly id: string;
readonly name: string;
readonly detail: string;
readonly principal: string;
readonly spans: number;
readonly processes: number;
readonly hosts: number;
readonly durationMs: number;
readonly error: boolean;
};
@ -20,7 +22,7 @@ type Summary = {
type Row = {
readonly name: string;
readonly depth: number;
readonly process: string;
readonly host: string;
readonly offsetMs: number;
readonly durationMs: number;
readonly attributes: readonly string[];
@ -35,14 +37,18 @@ type Answer<T> = { readonly kind: "ok"; readonly value: T } | { readonly kind: "
/** The viewer: the list, and the chosen trace's tree. */
export function TraceViewer() {
const [list, setList] = useState<Answer<readonly Summary[]>>();
const [principal, setPrincipal] = useState("");
const [chosen, setChosen] = useState(fragment);
const [trace, setTrace] = useState<Answer<{ readonly spans: readonly Row[] }>>();
useEffect(() => {
void read<readonly Summary[]>("/api/traces").then(setList);
const follow = () => setChosen(fragment());
addEventListener("hashchange", follow);
return () => removeEventListener("hashchange", follow);
}, []);
useEffect(() => {
const query = principal === "" ? "" : `?principal=${encodeURIComponent(principal)}`;
void read<readonly Summary[]>(`/api/traces${query}`).then(setList);
}, [principal]);
useEffect(() => {
setTrace(undefined);
if (chosen === "") return;
@ -56,6 +62,12 @@ export function TraceViewer() {
"nav",
null,
h("h2", null, "Traces"),
h("input", {
className: "traces-filter",
placeholder: "principal",
value: principal,
onChange: (event: { target: { value: string } }) => setPrincipal(event.target.value),
}),
...(list?.value ?? []).map((summary) =>
h(
"a",
@ -65,11 +77,11 @@ export function TraceViewer() {
className: summary.id === chosen ? "traces-chosen" : undefined,
},
h("span", { className: summary.error ? "traces-error" : undefined }, summary.name),
h("small", null, summary.detail),
h("small", null, [summary.detail, summary.principal].filter((part) => part !== "").join(" · ")),
h(
"small",
null,
`${count(summary.spans, "span")} · ${count(summary.processes, "host")} · ${ms(summary.durationMs)}`,
`${count(summary.spans, "span")} · ${count(summary.hosts, "host")} · ${ms(summary.durationMs)}`,
),
),
),
@ -78,19 +90,19 @@ export function TraceViewer() {
);
}
/** One trace as a tree: a colour per host (the process that recorded the span), lengths on the trace's time line. */
/** One trace as a tree: a colour per host (the host that recorded the span), lengths on the trace's time line. */
function Tree({ rows }: { readonly rows: readonly Row[] }) {
const total = Math.max(...rows.map((row) => row.offsetMs + row.durationMs));
const hosts = [...new Set(rows.map((row) => row.process))];
const hosts = [...new Set(rows.map((row) => row.host))];
return h(
"ol",
{ className: "traces-tree" },
...rows.map((row, index) => h(Span, { key: index, row, host: hosts.indexOf(row.process), total })),
...rows.map((row, index) => h(Span, { key: index, row, colour: hosts.indexOf(row.host), total })),
);
}
/** One span: its host, name and length, its bar, then its attributes, events, links and error. */
function Span({ row, host, total }: { readonly row: Row; readonly host: number; readonly total: number }) {
function Span({ row, colour, total }: { readonly row: Row; readonly colour: number; readonly total: number }) {
const indent = { paddingLeft: `${row.depth * 1.1}rem` };
// Inline styles are set through the DOM by React, which the page's CSP allows (it refuses style attributes in HTML).
const bar = {
@ -103,14 +115,14 @@ function Span({ row, host, total }: { readonly row: Row; readonly host: number;
h(
"div",
{ className: "traces-span", style: indent },
h("span", { className: `traces-host traces-host-${host % 4}` }, `host ${host + 1}`),
h("span", { className: `traces-host traces-host-${colour % 4}` }, row.host),
h("b", null, row.name),
h("span", { className: "traces-ms" }, ms(row.durationMs)),
),
h(
"div",
{ className: "traces-line" },
h("span", { className: `traces-bar traces-host-${host % 4}`, style: bar }),
h("span", { className: `traces-bar traces-host-${colour % 4}`, style: bar }),
),
h(
"div",

View file

@ -1,7 +1,8 @@
// The todo app's start: what the app decides itself, given to the kit's `startServer`: its name and data directory,
// its API, its facet kinds, its principals, its one worker kind (a list), its page and its terminal client, and its
// telemetry store (one SQLite file in the data directory, shared by the server and the workers). The kit does the rest. The examples call `startTodo` with a copy of `plugins/` they may edit; a real app's start script calls
// it with the data directory from its command line and prints the owner link.
// telemetry store (one SQLite file in the data directory, which the server writes for every process). The kit does the
// rest. The examples call `startTodo` with a copy of `plugins/` they may edit; a real app's start script calls it with
// the data directory from its command line and prints the owner link.
import { cp, mkdtemp } from "node:fs/promises";
import { tmpdir } from "node:os";
@ -12,8 +13,8 @@ import * as v1server from "@todo/app/v1/server";
import type { Context } from "../../../src/context/index.ts";
import { type App, startServer } from "../../../src/node/app/index.ts";
import { type PageAssets, pageScripts } from "../../../src/node/surfaces/index.ts";
import type { TodoPrincipal } from "./src/principals.ts";
import { type TodoTelemetry, telemetryFile } from "./src/telemetry.ts";
import { principalAttributes, type TodoPrincipal } from "./src/principals.ts";
import type { TodoTelemetry } from "./src/telemetry.ts";
/** The app's API: its name, its versions (newest first), and this process's instances of its modules. */
export const API = {
@ -54,14 +55,15 @@ export function startTodo(
api: API,
kinds: KINDS,
principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", list: address.key }) },
// The server's spans and metrics; each list's worker opens the same file (`worker.ts`).
// The store the server writes: its own spans and metrics, and what the workers and recording pages send.
telemetry: telemetry.store,
// Who a span was for: a short key and a user's name (each worker's `startWorker` gets the same hook).
principalAttributes,
workers: [
{
kind: v1.List,
entry: fileURLToPath(new URL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
env: { TODO_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// Who may reach a list's worker: everyone signed in, and the server itself.
authorize: (principal) => principal.kind !== "worker",

View file

@ -1,25 +1,34 @@
// The todo app's trace reader: a script reading the telemetry store of a data directory (`telemetry.sqlite`) through
// the store's read interface, while the app runs or after. Without a trace id it prints the newest traces (those whose
// first span is `--name`, when given) and the bytes on the links; with one, that trace as the renderer prints it.
// first span is `--name`, and with a span for `--principal`, when given) and the bytes on the links; with one, that
// trace as the renderer prints it.
// Run from packages/chord:
// node --conditions=source --experimental-strip-types test/examples/61-tutorial/trace.ts <data directory> [--name <span>]
// node --conditions=source --experimental-strip-types test/examples/61-tutorial/trace.ts <data directory> [--name <span>] [--principal <key>]
// node --conditions=source --experimental-strip-types test/examples/61-tutorial/trace.ts <data directory> <trace id>
import { parseArgs } from "node:util";
import { renderTrace } from "@earendil-works/pi-telemetry/render";
import { openTelemetry, telemetryFile } from "./src/telemetry.ts";
import { openTelemetryStore } from "../../../src/telemetry-sqlite/node.ts";
import { telemetryFile } from "./src/telemetry.ts";
const { values, positionals } = parseArgs({ allowPositionals: true, options: { name: { type: "string" } } });
const { values, positionals } = parseArgs({
allowPositionals: true,
options: { name: { type: "string" }, principal: { type: "string" } },
});
const [dataDir, id] = positionals;
if (dataDir === undefined) throw new Error("usage: trace.ts <data directory> [--name <span>] [<trace id>]");
// A recorder of this process on the shared file, which records nothing: the reads are what the script uses.
const telemetry = await openTelemetry(telemetryFile(dataDir));
if (dataDir === undefined) {
throw new Error("usage: trace.ts <data directory> [--name <span>] [--principal <key>] [<trace id>]");
}
// The shared file, read-only: the app's processes may be writing it meanwhile.
const store = await openTelemetryStore(telemetryFile(dataDir));
try {
if (id !== undefined) {
const trace = await telemetry.store.getTrace(id);
const trace = await store.getTrace(id);
console.log(trace === undefined ? `no trace ${id}` : renderTrace(trace.spans));
} else {
// Complete traces, newest first; a trace's first span (its root, by start time) names it.
const { traces } = await telemetry.store.findTraces({ name: values.name, limit: 10 });
// Complete traces, newest first; a trace's first span (its root, by start time) names it, and the app's
// `principalAttributes` hook named whom each span was for.
const attributes = values.principal === undefined ? undefined : { "chord.principal": values.principal };
const { traces } = await store.findTraces({ name: values.name, attributes, limit: 10 });
for (const { traceId, spans } of traces) {
const start = Math.min(...spans.map((span) => span.startedMs));
const length = Math.max(...spans.map((span) => span.endedMs)) - start;
@ -28,12 +37,12 @@ try {
}
// Metrics are stored per series and 10 s: a series is one set of attributes, here the direction.
for (const direction of ["sent", "received"]) {
const rows = await telemetry.store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
const rows = await store.findMetrics({ name: "chord.link.bytes", attributes: { direction } });
const bytes = rows.reduce((sum, row) => sum + row.sum, 0);
const messages = rows.reduce((sum, row) => sum + row.count, 0);
console.log(`chord.link.bytes ${direction}: ${bytes} bytes in ${messages} messages`);
}
}
} finally {
await telemetry.close();
await store.close();
}

View file

@ -1,10 +1,9 @@
// The todo app's worker entry: one process per list. `startWorker` opens the list's store in its directory, connects
// to the server and loads the plugins' worker facets. The worker records to the app's telemetry store, the file its
// server named in the environment: its methods called through the server continue the caller's trace.
// to the server and loads the plugins' worker facets. The worker records its spans in memory and sends them to the
// server, which stores them: its methods called through the server continue the caller's trace.
import { startWorker } from "../../../src/node/app/index.ts";
import { openTelemetry } from "./src/telemetry.ts";
import { principalAttributes } from "./src/principals.ts";
import { API, KINDS } from "./start.ts";
const telemetry = await openTelemetry(process.env.TODO_TELEMETRY!);
startWorker({ api: API, kinds: KINDS, telemetry: telemetry.store, onError: (error) => console.error("list:", error) });
startWorker({ api: API, kinds: KINDS, principalAttributes, onError: (error) => console.error("list:", error) });

View file

@ -1,5 +1,5 @@
// Telemetry across hosts (slice 9): a kit app (`start.ts`) whose server and tally workers record to one SQLite
// telemetry store in the data directory. The owner's client host calls a tally's counter: the call goes from the client
// Telemetry across hosts (slice 9): a kit app (`start.ts`) whose server records to a SQLite telemetry store in the
// data directory, and whose tally workers send what they record to the server. The owner's client host calls a tally's counter: the call goes from the client
// over the server's relay into the worker's method, which checks its argument in a span of its own (`ctx.trace`) and
// commits through a local store object. Each host starts its spans with its own adapter; the carriers on the wire make
// them one trace. The example prints that trace as the store renders it, and a metric the server recorded.
@ -18,7 +18,7 @@ import { openTelemetry, startTally, telemetryFile } from "./start.ts";
const dataDir = join(await mkdtemp(join(tmpdir(), "tally-")), "data");
// 1. The server records to the store in its data directory; each tally's worker opens the same file.
// 1. The server records to the store in its data directory; each tally's worker sends its spans to the server.
await mkdir(dataDir, { mode: 0o700 });
const telemetry = await openTelemetry(telemetryFile(dataDir));
const app = await startTally(dataDir, telemetry, context);
@ -42,11 +42,17 @@ const total = await telemetry.startSpan({ name: "tally.click" }, async (span) =>
});
console.log("3. the tally's total:", total);
// 4. Each process flushes its settled spans every second: wait for the worker's, flush the server's, then read the
// trace back from the store (the newest trace whose first span is the click).
await new Promise((resolve) => setTimeout(resolve, 1_500));
await telemetry.flush();
const { traces } = await telemetry.findTraces({ name: "tally.click", limit: 1 });
// 4. A worker sends its settled spans every second, the server writes every second: read until the worker's method is
// in the click's trace (the newest trace whose first span is the click).
const clicked = async () => {
await telemetry.flush();
return (await telemetry.findTraces({ name: "tally.click", limit: 1 })).traces;
};
let traces = await clicked();
while (!traces[0]?.spans.some(({ name }) => name === "tally.check")) {
await new Promise((resolve) => setTimeout(resolve, 100));
traces = await clicked();
}
console.log("4. the trace across the client, the server and the worker (links name the root span they point to):");
// The roots a trace links to (the worker's start, an object's activation) are traces of their own: named here.
const names = new Map<string, string>();

View file

@ -1,6 +1,5 @@
// The tally app's start: `startServer` with the app's telemetry adapter, a SQLite telemetry store in the data
// directory. Each process records to the store it is given: here every process opens the same file (`telemetry.sqlite`
// in the data directory; its workers get the path in their environment), so one trace reads tab → server → worker.
// The tally app's start: `startServer` with the app's telemetry store, a SQLite file in the data directory. The server
// is its only writer: each worker sends what it records to the server, so one trace reads tab → server → worker.
import { join } from "node:path";
import { fileURLToPath } from "node:url";
@ -22,7 +21,7 @@ export function telemetryFile(dataDir: string): string {
return join(dataDir, "telemetry.sqlite");
}
/** Opens the SQLite telemetry store at `path` (each process its own recorder on the shared file). */
/** Opens the SQLite telemetry store at `path` (the server's). */
export async function openTelemetry(path: string): Promise<SqliteTelemetry> {
return createSqliteTelemetry(await openNodeSqliteDatabase(path));
}
@ -48,7 +47,6 @@ export function startTally(
kind: v1.Tally,
entry: fileURLToPath(new URL("./worker.ts", import.meta.url)),
execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"],
env: { TALLY_TELEMETRY: telemetryFile(dataDir) },
start: () => ({}),
// The owner reaches a tally through the server, which relays.
authorize: (principal) => principal.kind === "owner" || principal.kind === "server",

View file

@ -1,9 +1,8 @@
// The tally app's worker entry: one process per tally, recording to the app's telemetry store (the file its server
// named in the environment). Its methods called through the server continue the caller's trace: the carrier travels on
// the wire.
// The tally app's worker entry: one process per tally. It records in memory and sends its spans and metrics to the
// server, which stores them (the kit does it when the server has a telemetry store). Its methods called through the
// server continue the caller's trace: the carrier travels on the wire.
import { startWorker } from "../../../src/node/app/index.ts";
import { API, KINDS, openTelemetry } from "./start.ts";
import { API, KINDS } from "./start.ts";
const telemetry = await openTelemetry(process.env.TALLY_TELEMETRY!);
startWorker({ api: API, kinds: KINDS, telemetry, onError: (error) => console.error("worker:", error) });
startWorker({ api: API, kinds: KINDS, onError: (error) => console.error("worker:", error) });