diff --git a/AGENTS.md b/AGENTS.md index 216a68b8b..eda34ed1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,9 +45,10 @@ Binding for Chord v2 (`packages/chord/src/`) and its tests, when writing and whe ## Commands - After code changes (not docs): `npm run check` (full output, no tail). Fix all errors, warnings, and infos before committing. Does not run tests. +- `apps/p2` is outside the root check and the root tests. After p2 changes also run `cd apps/p2 && npm run check`; p2's suite is `cd apps/p2 && npm test` (TUI tests need Node 26, browser tests need `CHROMIUM_BIN`). - Run `npm run build` whenever something needs built output (`dist/`): tests or benchmarks that import built packages, Durable's workerd suite. The full suites of Chord and Durable need it: Chord's tests that start Node processes or bundle with esbuild load workspace packages from `dist/` (the rest run from source by its vitest config), and Durable imports pi-ai's `utils/*`, which have no source export. Never use it to type check or lint: that is `npm run check`, the preferred check. - Never run `npm test` unless requested by the user. -- Never run the full vitest suite directly: it includes e2e tests that activate when endpoint/auth env vars are present. For all non-e2e tests, run `./test.sh` from the repo root. Otherwise run specific tests from the package root: +- Never run the full vitest suite directly: it includes e2e tests that activate when endpoint/auth env vars are present. For all non-e2e tests, run `./test.sh` from the repo root. It runs `packages/*` only. Otherwise run specific tests from the package root: - Vitest: `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/specific.test.ts` - `packages/tui` (`node:test`): `node --test test/specific.test.ts` - If you create or modify a test file, run it and iterate on test or implementation until it passes. diff --git a/apps/p2/.gitignore b/apps/p2/.gitignore new file mode 100644 index 000000000..546b0db76 --- /dev/null +++ b/apps/p2/.gitignore @@ -0,0 +1,3 @@ +# Vite dependency caches from before the web UI was bundled with esbuild; safe to delete. +.vite/ +.vite-test-*/ diff --git a/apps/p2/DEV.md b/apps/p2/DEV.md new file mode 100644 index 000000000..2474e436b --- /dev/null +++ b/apps/p2/DEV.md @@ -0,0 +1,58 @@ +# p2 development + +See [p2 notes](docs/p2-notes.md) for architecture, reloads and testing; [README](README.md#tui-isolation) defines +the TUI threat model. Production/data directories must not be used for tests. Commit nothing without approval. + +## Server and TUI + +The server supports Node 22.19+. The launcher requires Node >=26.5.1; slayer has Node 26.11.1 at +`~/.local/bin/node26`. This wrapper supplies the libraries required by that binary. From `apps/p2`: + +```sh +~/.local/bin/node26 --import ./src/source.ts src/hosts/tui.ts +~/.local/bin/node26 --import ./src/source.ts src/hosts/tui.ts https://your-domain +npm run check +~/.local/bin/node26 --import ./src/source.ts --test --test-concurrency=1 test/p2/tui*.test.ts +``` + +To use `npm run tui`/`npm test` with this Node, put its binary directory first in PATH and supply its library path: + +```sh +PATH="$HOME/node-v26.11.1-linux-x64/bin:$PATH" LD_LIBRARY_PATH="$HOME/node26-lib" npm run tui +PATH="$HOME/node-v26.11.1-linux-x64/bin:$PATH" LD_LIBRARY_PATH="$HOME/node26-lib" npm test +``` + +The server builds one ESM bundle through esbuild and serves it only to web-admitted owners/share guests. Facets +are never imported from the visitor's checkout. Successful server plugin reloads rebuild/publish the bundle; +the launcher swaps to a fresh worker only after readiness and terminates its predecessor. TUI-only source/import +changes count as dirty plugins. TUI imports gate facet inclusion without expanding server/worker selection. +Keybindings, theme and display settings are guest-local: the launcher reads pi's `~/.pi/agent` files +(`PI_CODING_AGENT_DIR`) and `~/.pi/p2-tui/prefs.json`, validates them and posts data to the worker, watching for +changes; `/tui.json` (`TuiPreferences`) holds only server defaults below them. Launcher `config.json` +holds only per-origin cookies. + +## Security boundary + +Do not move authentication, sockets, raw terminal management or machine access into downloaded code. The launcher +recognizes a small message protocol; unknown capabilities are denied. Only opaque chord frames use its fixed-origin +socket. Download manifests/paths, byte bounds and version hashes are validated; redirects are refused. + +The trusted Worker bootstrap must drop `fs.read`, `fs.write`, `worker`, `net`, `child`, `addon`, `wasi` and `inspector` +before ANY downloaded module evaluation. Node's permission-mode workers have implicit module-read grants until +explicitly dropped. Bundle import is from an in-memory data URL. Empty env, captured/discarded stdout/stderr, +V8 limits and cheap process/V8/worker-surface taming are required. Never expose parentPort or the Worker handle. +Disable the entire fs/fs-promises function facade too: inherited numeric descriptors can bypass path-based +permission checks, including raw fd reads/writes. BroadcastChannel and environment-data APIs must not communicate +with other host workers. These restrictions precede the bundle import and are tested through built-in aliases. + +All downloaded output is untrusted: it must pass the launcher's bounded, complete-sequence terminal allowlist. +Never forward raw worker output, logs, exceptions, OSC/clipboard/title/hyperlinks or image protocols. Local file, +clipboard, browser, editor and job-control capabilities exist only as launcher actions (`src/hosts/tui-actions.ts`): +each needs an explicit, launcher-owned user grant (the visitor's own key, or a confirmation in the launcher-drawn +consent screen of the exact data) with validated data, never a server-provided path, a credentialed URL or a +simulated prompt. Adding one follows p2-tui-port.md sections 3.1-3.5. + +Denial of service is owner-accepted. The watchdog samples RSS/ArrayBuffers and terminates the worker over its caps, +but sampling/V8 limits cannot enforce a hard native-memory quota; memory exhaustion can affect the host. There is +no OS sandbox. Runtime vulnerabilities remain out of scope. Keep Node patched and rerun the escape tests after +runtime changes. Tests use fake providers, disposable directories and port 0; no production or paid provider calls. diff --git a/apps/p2/README.md b/apps/p2/README.md new file mode 100644 index 000000000..b7812bc27 --- /dev/null +++ b/apps/p2/README.md @@ -0,0 +1,214 @@ +# p2 + +A self-hosted Pi Durable UI built from independent [Chord](../../packages/chord) plugins. Owners have full agent and +shell control on that server and its workers; this is not server-side filesystem isolation. Downloaded TUI facets run in a +locked-down worker on the visitor's machine. + +## Run + +Requires Node 22.19+ and the monorepo's installed dependencies and model data. From the repository root: + +```sh +npm install --ignore-scripts +npm run hydrate:model-data +cd apps/p2 +npm start +``` + +Open http://127.0.0.1:8788. `PORT` changes the port; `P2_DATA_DIR` changes the default data directory `~/.pi/p2`. +`P2_FAKE=1 npm start` uses a fake provider for demos, but tools still act on real files. +See [development notes](docs/p2-notes.md) for dev mode, architecture and testing. +Code mode scripts can orchestrate one-shot agents (`tools.agent`); [docs/workflows.md](docs/workflows.md) has workflow +patterns with example scripts (fan-out with adversarial verification, drafts plus a judge, fix until a check passes, +search until nothing new). + +## Using the owner UI from another device + +The server listens only on loopback. Use an HTTPS tunnel or reverse proxy preserving the public Host header, and +list its hostname in `P2_ALLOWED_HOSTS` (comma-separated for several hosts). + +1. On the host, with `PORT` matching its running server: + + ```sh + npm run owner-link -- https://your-domain + ``` + + The base defaults to `P2_PUBLIC_URL`, or `http://127.0.0.1:$PORT` (8788 when unset). The command uses the server's + direct device-only loopback path; it does not need a browser or read the signing key. +2. Open the printed `https://your-domain/owner#` link on your other device. The page removes the fragment, + POSTs its token on the same origin, then redirects to `/`. The fragment never reaches HTTP/proxy logs. + +Links work once and expire after ten minutes. Treat their output as a private credential: do not publish it or +put the token in a query/path. You can also use **Devices → Copy token** on the localhost owner UI, or the private +`/owner-token.txt` startup token, and paste it at `/owner`. + +Owners get distinct signed, HttpOnly, SameSite=Lax cookies, Secure on a remote HTTPS origin. Cookies last 30 days +(server-side expiry too), are port-scoped, and survive restarts through the private `owner-key`. Signed-in devices +are remote owners (`access: "device"`), so provider login and SSH-key upload work but owner-token minting does not. +Only the trusted localhost device owner can mint invitations. Guest sharing is independent and session-limited. +Radius remains a separate authenticated owner transport; a proxy marker is never authentication. + +**Devices → Sign out on this device** revokes this cookie, including its live connections; other devices stay +signed in. **Sign out all devices** rotates the signing key, revokes all associated cookies/connections and +outstanding links. Individual revocations persist in private `owner-revoked.json`. Already-admitted agent work +continues. The host's inherent localhost/device access cannot be revoked by signing out remote devices. + +The first deployment of signed cookies rotates the former raw-key cookie secret once (`owner-cookie-version`): +old raw-key cookies intentionally stop working. Generate fresh owner links after deploying. Outstanding invitations +do not survive server restarts; a fresh headless token replaces `owner-token.txt`. Compatible plugin reloads retain them. + +## Providers and SSH keys + +**Providers** is available to every owner (local, associated device or Radius), never guests. Provider-owned +pi-ai flows support API keys and OAuth; credentials stay in the server's private `auth.json`. Remote connections prefer +device-code methods where offered. For a browser-callback flow, complete sign-in in the provider tab and paste its +code or final redirect URL into Providers. If OpenAI ChatGPT lands on an unreachable localhost page, copy its entire +address from the address bar and paste it; the page need not load. Anthropic displays a code to paste. The provider's +registered callback URL is unchanged. + +**Envs → Upload private key** also works for every owner, locally or remotely, never guests. Keys must be passphrase-less +and at most 16 KiB; they are stored privately under `/ssh/keys/`, never returned to a browser. Key generation +and fetching an existing public key are available to every owner too, never guests. See [remote environments](docs/remote-envs.md). + +API keys, pasted login codes and uploaded SSH keys pass through your tunnel or relay. Use a trusted transport. +Keep the data directory, owner-key/tokens/revocations, provider credentials and SSH keys private; never commit them. + +## Deploy behind a proxy (no HTTP basic auth) + +Keep p2 loopback-only. Preserve the public Host header and forward WebSocket upgrades. Configure +`P2_ALLOWED_HOSTS=your-domain` and `P2_PUBLIC_URL=https://your-domain`. Overwrite `X-P2-Proxy-Secret` on **every** +request with a constant marker, for example `1`. The historical header name remains, but its value has no authority +and `P2_PROXY_SECRET` is no longer used. Any marked/forwarded localhost request is rejected rather than receiving +device privileges, even when a Unix-socket bridge makes its network peer `127.0.0.1`. Allowed public hosts are +remote regardless of the marker; without an owner/guest cookie visitors get only a minimal sign-in page and no +application assets, sessions, WebSockets or file/upload access. Share links still work without owner sign-in. + +Example Caddyfile behind an existing HTTPS front end (the Unix-to-loopback bridge is unchanged): + +```caddyfile +:80 { + reverse_proxy unix//p2/p2.sock { + header_up X-P2-Proxy-Secret 1 + flush_interval -1 + } +} +``` + +Remove `basic_auth` and the old secret injection. Do not rewrite the public Host to localhost. Ensure the public +front end actually uses HTTPS; p2 treats allowed public hosts as HTTPS origins. Restart p2 to apply runtime/auth +changes, then reload the proxy and mint a fresh owner link. Do not run competing servers against the same data dir. + +## TUI sign-in + +The TUI launcher requires **Node >=26.5.1**; the server still supports Node 22.19+. On slayer use Node 26.11.1: +`PATH="$HOME/node-v26.11.1-linux-x64/bin:$PATH" LD_LIBRARY_PATH="$HOME/node26-lib" npm run tui` +(or invoke `~/.local/bin/node26 --import ./src/source.ts src/hosts/tui.ts`). + +```sh +npm run tui # device owner on http://127.0.0.1:$PORT (8788 when unset) +npm run tui -- https://your-domain # prompts for an owner link if no valid cookie is saved +npm run tui -- https://your-domain --owner-link 'https://your-domain/owner#' +``` + +Direct loopback auto-signs in as the device owner without a link prompt; a proxy marker cannot use that path. +For remote URLs (HTTPS required), get an owner link on the host, then pass `--owner-link` or paste it at the prompt. +The TUI stores only its cookie per origin in `~/.pi/p2-tui/config.json` (0600, directory 0700; +`P2_TUI_CONFIG_DIR` overrides it). It never stores the consumed invitation. `P2_TUI_OWNER_LINK` is an optional +alternative to an argument. Links must match the selected origin and put the token in the fragment only. +Expired/revoked credentials prompt again on the next launch. Basic-auth flags, environment and saved passwords +are no longer used. If an active remote TUI is revoked, restart it and sign in with a fresh link. + +### TUI isolation + +The server esbuilds one self-contained ESM containing pi-tui, the Chord client, the runtime and its selected +`plugins/*/tui.ts` facets, only when their TUI dependencies are already selected (they never expand backend selection). +The launcher fetches the authenticated `/__p2_tui` manifest and hash-addressed bundle; +it never discovers or imports TUI facets from the visitor's checkout. Successful plugin reloads rebuild the bundle. +A changed hash starts a fresh worker; only after readiness does the launcher swap, restore UI state and terminate +the old worker. `/reload` requests a fresh worker too. Network drops reconnect to the same server. + +The unrestricted launcher alone owns terminal input/resize, WebSocket, sign-in and local cookie storage. A plain +Worker runs with `execArgv: ["--permission"]`, `env: {}` and V8 resource limits. Its trusted bootstrap drops +`fs.read`, `fs.write`, `worker`, `net`, `child`, `addon`, `wasi` and `inspector` **before** importing the bundle +from an in-memory data URL. Worker stdout/stderr are captured and discarded. Heap snapshots/object queries, +worker environment-data/BroadcastChannel access, parentPort exposure and process-signaling/report controls are also +disabled. The entire worker fs/fs-promises function facade is closed: inherited numeric descriptors must not read +stdin or bypass stdout/stderr capture. Permission drops still protect JSON/require/file module loading. +Keybindings, theme and display settings are the visitor's own: the launcher reads pi's `~/.pi/agent/keybindings.json`, +`settings.json` and `themes/*.json` plus `~/.pi/p2-tui/prefs.json`, validates a whitelisted subset and posts it to the +worker as data (no paths, env or commands); the server's `TuiPreferences` (`/tui.json`) only supplies +defaults below them and never privileged bindings. The worker never reads files. Clipboard writes, opening URLs, downloads, +the external editor and Ctrl+Z are launcher actions under a grant (the visitor's own key, or a confirmation in a +launcher-drawn screen showing the exact data); server code cannot pick a visitor path, read the clipboard or open a +browser by itself. + +All worker output passes a tight 64 KiB/frame terminal filter: printable text, CR/LF/tab, SGR, cursor/erase, +cursor visibility, synchronized output, bracketed paste, pi-tui's alternate screen and scroll region. OSC +(including clipboard/title/hyperlinks), DCS, APC/PM/SOS, kitty graphics, C1 controls, unknown modes and incomplete +escapes are stripped. Chord frames are relayed unchanged over the launcher's fixed-origin connection. + +**Accepted risk: denial of service.** Malicious server code may crash or exhaust its TUI process. V8 limits bound +heap, not every native/ArrayBuffer allocation; a launcher watchdog samples process RSS and ArrayBuffers every +100 ms and terminates the worker above 512 MiB RSS or 128 MiB main-thread ArrayBuffers. RSS covers all threads; +ArrayBuffer accounting is thread-local. Sampling is not a hard OS quota: rapid allocations/native work can beat +it, and memory exhaustion can affect the host. Node/runtime vulnerabilities remain out of scope; keep Node patched. +There is no OS sandbox. The intended authority boundary denies machine reads/writes, network, child processes, +inherited environment and terminal escapes; a connected server retains full authority over its own workers. + +### Owner testing on slayer + +From `/home/badlogic/pi-private/apps/p2` (no server restart needed): + +```sh +~/.local/bin/node26 --import ./src/source.ts src/hosts/tui.ts +# Loopback http://127.0.0.1:8788: automatic device owner, no prompt. + +npm run owner-link -- https://p2.mariozechner.at +~/.local/bin/node26 --import ./src/source.ts src/hosts/tui.ts https://p2.mariozechner.at +# Paste the private printed link at the prompt. Ctrl+D exits. +``` + +Do not publish the owner link or paste it into logs. A valid saved remote cookie skips the prompt on later launches. + +## Configuration + +Edit `/settings.json` as a file; there is no Settings editor UI. Valid changes hot-reload without a +restart, and in-flight requests keep their configuration. Set the default model and thinking level with +**Make default for new sessions** in the model picker. `ownerName` (1 to 40 characters, no control characters, `<` +or `>`; default "Owner") is the name your messages carry from then on; messages already written keep theirs. + +### Voice mode + +Voice mode talks to the agent hands-free: the microphone button in the composer (owner only) streams your speech to a +speech service, each finished utterance becomes a message, and the agent's answers are read aloud sentence by sentence. +Talking over an answer stops it; "stop" also stops the agent. While it is on, the conversation's prompt asks for short +spoken answers. The button appears only when `/voice.json` names a speech service: + +```json +{ "url": "https://voice.mariozechner.at", "token": "" } +``` + +p2 watches the file: writing, replacing or removing it shows or hides the button in open pages at once, and the +service's engines and voices are fetched again on each change (the token stays on the server). + +The speech service is `voice-server/` (Python, CPU only). Speech to text: Silero VAD and Parakeet TDT 0.6B v3, with +750 ms of audio before the detected speech start kept (`VOICE_VAD_PREROLL_MS`) so quiet first words survive. +Speech: selectable engines, listed with their voices by `GET /voices`: Supertonic 3 (default; voices F1-F5, M1-M5), +Kokoro and Kyutai Pocket TTS. Voice ids are `-`, for example `supertonic-F1` or `pocket-alba`. The +`/speak/stream` WebSocket sends 16-bit PCM chunks as they are generated: Pocket TTS streams ~80 ms chunks (first audio +after ~50-80 ms on slayer), Supertonic and Kokoro send a chunk per sentence (~0.2 s). `POST /speak` returns a WAV. +`server.py`'s docstring specifies the protocol. On slayer the service runs from `~/voice.mariozechner.at` (`docker +compose up -d`, token in `.env`); rebuild with `docker build -t p2-voice apps/p2/voice-server`. Browsers never reach +it directly: p2 proxies `/voice/listen` (WebSocket) and `/voice/speak` for the signed-in owner. + +## Tests + +```sh +npm run check +npm test +``` + +Tests use fake providers, temporary data/workspaces and port 0. Browser tests use `CHROMIUM_BIN`; daemon/SSH tests +skip when their prerequisites are unavailable. On Android (Termux) browser tests are skipped unless `P2_BROWSER_TESTS=1`: +Chromium's processes plus a test's p2 server and workers exceed Android's limit of 32 child processes, and its phantom +process killer then kills processes, the running p2 included. `P2_BROWSER_TESTS=0` skips them elsewhere too. No real provider logins or paid requests are needed. diff --git a/apps/p2/biome.json b/apps/p2/biome.json new file mode 100644 index 000000000..8b58965a1 --- /dev/null +++ b/apps/p2/biome.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://biomejs.dev/schemas/2.3.5/schema.json", + "formatter": { + "enabled": true, + "indentStyle": "tab", + "indentWidth": 3, + "lineWidth": 120 + }, + "linter": { + "enabled": true, + "domains": { + "react": "recommended" + }, + "rules": { + "recommended": true, + "style": { + "noNonNullAssertion": "off" + } + } + }, + "files": { + "includes": [ + "src/**", + "plugins/**", + "!plugins/terminals/vendor", + "test/**", + "!test/fixtures/v1-data/data", + "scripts/**", + "*.json", + "!package-lock.json" + ] + } +} diff --git a/apps/p2/docs/changes.md b/apps/p2/docs/changes.md new file mode 100644 index 000000000..2b8955737 --- /dev/null +++ b/apps/p2/docs/changes.md @@ -0,0 +1,531 @@ +# Chord and p2: decided changes + +Changes decided while walking through Chord's concepts. Each entry says what changes, why, and what it touches. The +loader and reload work is in `plugin-loading.md` (12.1, 14); the open redesign of resources, listeners and lifecycles is +still being discussed and is not listed here until decided. + +## 1. Terminology: facet > object type > object, reference + +- **facet:** a plugin's code for one surface (`server.ts`, `worker.ts`, `web.tsx`, `tui.ts`); it serves object types. +- **object type:** declared by a contract: name, input (the address), members. +- **object:** one instance of an object type (Chord's code calls it a record). +- **reference:** what `get` returns (Chord's code: handle; the bookkeeping behind it: hold). + +Code and docs (packages/chord, apps/p2) are renamed to these words: `record` → object, `Handle`/`Hold` → reference, +behavior → the object's code, resource spec → resource declaration, epoch → code version. + +**An object's variant is named `variant`, not `kind`.** Factories return `{ variant: "editor", … }`, callers check +`ref.variant === "editor"`. Other `kind`s stay: member descriptions (`kind: "method" | "state" | "opaque"`, +`contract.ts:58-60`, part of the fingerprint), the link peer kind (`"host" | "client"`, hello frame), principals, +`Author.kind`, app types. + +Sites: `RESERVED_MEMBERS` (`contract.ts:98`: `kind` → `variant`), `kindOf` (`record.ts`) and `VariantBehavior` +(`contract.ts:253`), the shape frame's `kind` (`wire.ts:79`, read at `connection.ts:864,990-994`), +`hosts/plugins.ts:295`, and every `ref.kind` / `{ kind: "" }` in plugin factories and UI code. + +**Wire.** Renaming a frame field breaks old peers: a stale browser tab or TUI client would read no variant and fall back +to `LOCAL_VARIANT` (`connection.ts:990-994`), so every call fails with `not_in_variant`. The link hello +(`connection.ts:332`) gets a protocol version; on a mismatch the client is told to reload instead of continuing. + +## 2. The object type says where it lives; callers stop passing `at` + +**Today.** A caller names the host with every `get`: `get(Workspace, { session, conversation }, { at: \`session:${session}\` })`. +p2 has 59 literal `at: "server"` and 26 `` at: `session:${id}` `` (built by hand from strings), plus `at` passed along +as a value: the `at` option of `useObject` and `holdLingering` (`src/runtime/web.tsx:174,178`, +`src/runtime/linger.ts:63`), the positional `at` of the TUI `watch` (`src/runtime/tui.ts:84-109`), +`src/web/file-completion.tsx:59`, and several hundred in tests. + +**Problems.** + +- Stringly typed: a typo (`"sever"`, `"sesion:"`) shows only at runtime. +- Where an object lives is knowledge every caller must carry. Moving a type from the server to the worker breaks every + caller, at runtime (`unknown type`). +- Redundant: the session is usually already in the input; callers write it twice. + +**Change.** The contract declares its host declaratively, through typed helpers. A helper carries a static pattern +(`server`, `session:*`) and a resolver from the input to the host name: + +```ts +export const Workspace = remote("workspace", { + input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + host: sessionHost((input) => input.session), // pattern "session:*" + … +}); +export const Sessions = remote("sessions", { input: Type.Object({}), host: serverHost, … }); // pattern "server" + +useObject(Workspace, { session, conversation }); // no `at` +ctx.get(Sessions, {}); +``` + +- One place knows where a type lives: its definition. Host names come from the helpers, never from free strings. +- Local types (same host only) have no `host`. +- The resolver runs on the normalized input (Chord normalizes before resolving), so defaults count. +- `serve` checks the pattern against `host.name` ("Workspace is declared for `session:*`, but `server` serves it"). This + covers types served outside facets too (`WorkerBoundary`, `PluginRemoval`, `Harness`), so a misplaced type fails at + startup, not at the first call. +- The serving host checks, at acquisition, that the normalized input resolves to itself. A destination chosen by a + client is never trusted. +- Placement is part of the contract fingerprint. Moving a type is a breaking contract change: existing references get + `upgrade_required` and clients reload. References are not relocated live. + +Types served on session workers with an empty input get a `session` input (version bump) and check +`host.name === \`session:${session}\`` in the factory: `WorkerBoundary` (`src/runtime/worker.ts:174`, caller +`plugins/sessions/server.ts:173`), `SessionTransfer` (`plugins/sessions/contract.ts:207`, callers +`plugins/sessions/server.ts:291,300`), `PluginRemoval` (`src/hosts/plugins.ts:107`, callers +`src/hosts/server.ts:301,333`). + +**Touches.** Chord (via pi first): `host` helpers with pattern and resolver in `remote(...)` contracts, address +resolution without `at`, the check at `serve` and at acquisition, placement in the fingerprint; `at` stays only for the +transition. p2: `host` on every remote contract, the three input changes above, removing the 85 literal options and the +plumbed `at` (`useObject`, `holdLingering`, `linger.ts`, TUI `watch`, `file-completion.tsx`), tests. + +## 3. Typed principals, all with a `kind` + +**Today.** Chord types a principal as `unknown` (`packages/chord/src/old/types.ts:5`); factories receive `unknown` and p2 +narrows it with helpers (`isOwner`, `isGuest`, `isWorker` in `src/runtime/principals.ts`). Chord also invents principals +of another shape, without `kind`, outside the app's type: `{ host: true }` for a host's own code (`host.ts:37`), +`{ host: parentName }` for the parent's own requests as a child sees them (`connection.ts:651`; in a worker: +`{ host: "server" }`), `{ host: hostName }` as the default for a routed host calling back (`host.ts:870`). Chord +recognizes the host principal by object identity (`principal === this.principal`, `host.ts:367,375,411`). Six places in +p2 cast by hand: five check `(principal as { host?: unknown }).host !== "server"` +(`plugins/search/worker.ts:167`, `plugins/conversations/worker.ts:124`, `plugins/sessions/worker.ts:67`, +`src/runtime/worker.ts:192`, `src/hosts/plugins.ts:222`), one checks `.host === true` (`mayReach`, +`src/runtime/server.ts:159`). p2's owner principal is confusing: `via: "device"` means the local machine, +`via: "relay"` covers both Radius and remote cookie devices, `device` is optional, and the owner has no name (`authorOf` +returns "Owner", the web prints the literal "Owner": `src/web/Chat.tsx:742,772`). + +**Change, Chord (via pi first).** The principal type is a type parameter of the host (`createHost

(...)`); factories, +`ctx`, `principalKey` and `invalidatePrincipal` receive `P` and TypeScript narrows on `kind`. Chord invents no +principals: the app supplies the principal of the host's own code (`createHost({ self: … })`) and the one the parent's +own requests carry (`host.connect(link, { principal: … })`); relayed requests keep the caller's principal; +`route.principal` is mandatory (no `{ host }` default). Chord compares the host principal by `principalKey`, not by +object identity, so an equal principal never creates a second object. + +**Change, p2.** + +```ts +/** The person running this p2 (one today). */ +export type OwnerPrincipal = + | { readonly kind: "owner"; readonly access: "local" } // a browser on this machine (loopback), no owner cookie + | { readonly kind: "owner"; readonly access: "relay" } // via Radius, authenticated by the relay + | { readonly kind: "owner"; readonly access: "device"; readonly device: string }; // owner cookie, also on loopback +/** Someone who joined one session through a share link. */ +export type GuestPrincipal = { + readonly kind: "guest"; + readonly id: string; + readonly name: string; // self-chosen, unverified + readonly session: string; + readonly mode: "view" | "full"; + readonly revision: number; // grant revision: removal, link reset or archive invalidates older ones +}; +/** A session's worker calling the server. */ +export type WorkerPrincipal = { readonly kind: "worker"; readonly session: string; readonly generation: number }; +/** The server's own code calling into a session worker (fork, transfers, the conversation broker). */ +export type ServerPrincipal = { readonly kind: "server" }; +/** A host's own code calling objects on the same host. */ +export type HostPrincipal = { readonly kind: "host" }; +export type P2Principal = OwnerPrincipal | GuestPrincipal | WorkerPrincipal | ServerPrincipal | HostPrincipal; +``` + +- The five `host !== "server"` casts become `principal.kind !== "server"`, keeping their extra checks (`host.name`, + session); `mayReach`'s `.host === true` becomes `principal.kind === "host"`. +- `principalKey`: `owner:local`, `owner:relay`, `owner:device:`, `guest::r`, + `worker::`, `server`, `host`. +- **Owner name.** Not in the principal: a link's principal is stamped once at accept, and the key decides which tabs + share an object, so a name in it would go stale. A setting `ownerName` (default "Owner", validated) is read when + writing: `authorOf` gets it from the worker's init values or a server value. The web renders `author.name` instead of + the literal "Owner" (`src/web/Chat.tsx:742,772`). Names in Durable entries already written stay as written. +- "Revoke all remote owners" (today one pattern `{ kind: "owner", via: "relay" }`, `src/runtime/server.ts:72`) becomes + two propagating pattern calls, `access: "relay"` and `access: "device"` (patterns match equal fields; predicates do not + propagate to child hosts, `host.ts:125`). +- Call sites of `via`: stamping in `src/hosts/server.ts:418,426,427` (a device-cookie owner stays `access: "device"` on + loopback), the loopback owner cookie (`src/hosts/server.ts:537`), Radius (`plugins/radius/server.ts:54`), owner + cookies (`plugins/devices/server.ts:56`), revocation (`src/runtime/server.ts:72`). Checks of `via === "device"` + become `access === "local"`: sharing join (`plugins/sharing/server.ts:260`), login `remote` + (`plugins/login/server.ts:169`), owner-link minting and `canMint` (`plugins/devices/server.ts:115,170,175`). + +**Later, not planned.** Several people (individuals, teams, orgs): users with IDs and roles instead of one owner, +sessions owned by a user or team, `canView`/`canSend` by membership. That touches every `isOwner` check (some mean +"admin", some "may use this session") and the single-tenant data directory (`models.json`, `auth.json`). + +## 4. A `files` tool: send and open (low priority) + +**Today.** `send_file` (`plugins/send-file/tool.ts`) copies a snapshot and shows a download card +(`src/web/Chat.tsx:504`). The `read`, `write` and `edit` cards already link into the workspace ("Open in workspace", +`src/web/Chat.tsx:335-342,388,399,543`). What is missing is a tool that only points the user at a file or directory, +without reading or writing it. The workspace route can show one: +`{ workspace: { tab: "files", directory, file: { path, view: "file", line } } }` (`src/web/navigation.ts`). + +**Change.** One tool, `files`, with an `action`: + +- `send` (path, optional name): today's `send_file`. +- `open` (path, optional line): resolves the path with `env.canonicalPath` through the session's environment, then + checks that it is a file or directory inside the conversation's canonical cwd (the workspace shows nothing else; p2's + private data directory stays excluded). It copies nothing and returns + `details: { action: "open", path (canonical, cwd-relative), kind, line? }`; the web derives `directory` from `path`. + +Replay: Durable's replay policy is per registration, so the whole `files` tool stays `replay: "unsafe"` like +`send_file` (`plugins/send-file/tool.ts:20`), unless `send` becomes idempotent. + +Web: the card shows the file with "Open in workspace →", navigating to the workspace route above; the link is hidden for +view-only guests, whom the workspace refuses (`plugins/workspace/worker.ts:177`, `canSend`). Undecided: also open it +automatically when the call finishes while the conversation is open in that tab (client-side only, never when loading +history; only for owner and full guests). TUI: no docked workspace; open it in the file view or show the path. +`/downloads` lists `files` send calls as well as old `send_file` calls. + +**Open question: old names.** Durable stores tool names in the `pi.agent` selections (`tools`, +`packages/durable/src/harness/agent.ts`), and pending calls resolve by exact name (`packages/durable/src/harness/tool.ts:210`). +Subagents store `"send_file"` in their tool list (`CODING_TOOLS`, `plugins/subagents/tools.ts:31`), and models copy +calls from their history. An alias in the subagent registry is therefore not enough. Either keep `send_file` registered +as a hidden compatibility tool (old arguments, unsafe replay, not offered to new configurations), or migrate the +`pi.agent` selections. + +**Touches.** Old Durable entries are named `send_file`: the web card, the TUI renderers and `/downloads` +(`toolViews("send_file")`, `plugins/send-file/tui.ts:99`) keep rendering that name. Rename in `CODING_TOOLS` and the +tool descriptions (`plugins/subagents/tools.ts:31,279`, `plugins/agent/tool.ts:49`), the fake provider +(`plugins/models/fake.ts:22`) and tests. + +## 5. `lifetime` instead of `eager` + +**Today.** `host.serve(Type, factory, { eager: true })`: the host creates the object when the facet loads, `load` waits +until it is live unless `awaitEager: false`, and the host holds it until it closes. Only such objects may register into +registries (`record.ts:1505`). `eager: { input }` allows another input (`record.ts:50`), and any host-principal `get` of +an eager type with another input also gets the eager flag (`host.ts:410-411`). The name says when the object is created, +not what matters: how long it lives. + +**Change.** An explicit lifetime: + +| `lifetime` | Created | Ends | `linger` | +| --- | --- | --- | --- | +| `"referenced"` (default, when omitted) | at the first `get` of its address | after the last reference is released, plus `linger` | allowed (default 0) | +| `"host"` | when its facet loads; loading waits until it is live (see below) | when the host closes, or a reload removes its facet | not allowed | + +- `"host"` is allowed only for local types, and the object has exactly one address: an empty input (or only defaults), + the host principal. Other inputs are rejected; `eager: { input }` is dropped. Others reach it with `get`; their + references do not change its lifetime. +- Loading waits for `"host"` objects by default. An explicit option (today's `awaitEager: false`, renamed) defers this: + a worker uses it to avoid the startup deadlock with the server, whose link is admitted only after the worker reports + ready (`src/hosts/worker.ts:49-62,86-93`). +- A `"host"` object whose activation fails is retried at the next reload, as today (`reload.ts:586`). +- Registrations stay limited to `"host"` objects. +- Most of today's `eager: true` objects are `contribute` dummies for registrations (`src/runtime/contribute.ts`); 15 of + the 34 also own references, resources, listeners or tasks, so they stay objects. +- `lifetime: "host"` replaces `eager: true` one to one. Catalog is not eager today, and the server's models object + lingers 60 s (`plugins/models/server.ts:67`): making them `"host"` would be a deliberate behavior change, not part of + the rename. + +## 6. References re-acquire by themselves after their object goes away + +**Today.** A reference whose object is forcibly ended (its worker exited, a reload removed its type, access was +revoked) goes `gone` and stays dead; the caller must `get` again. p2 does that by hand: `useObject` +(`src/runtime/web.tsx:183`) re-acquires 500 ms after any `gone`, the TUI `watch` (`src/runtime/tui.ts:84`) retries with +backoff. For a lost **link** Chord already re-attaches remote references by itself (`connection.ts:939-941`, `connect`), +so the two cases behave differently. Revocation ends objects with code `gone` (`host.ts:704`, `connection.ts:441`) and +then closes the link (`connection.ts:445`; p2 passes no `restamp`): the browser's references go `loading` and wait for a +reconnect. p2 detects an ended share only through HTTP (`src/hosts/web/main.tsx:31-52`). + +**Change.** A reference re-acquires its object after a retryable loss, as it does after a lost link, with backoff. + +- **Decided by error code and reason, defined across the wire.** Final: `forbidden`, `revoked`, `invalid`, + `contract_mismatch`, `not_in_variant`, `upgrade_required`, `unavailable` with reason `unknown_type`. Retryable: link + loss, worker exit. +- **Revocation is final.** `invalidatePrincipal` ends references with the distinct reason `revoked`, including + provider-side and relayed holds; they are never re-acquired with the old principal. A link whose reconnect is refused + by authentication ends its references instead of leaving them `loading`. +- **Failing workers.** p2's server throws a `ChordError` with a reason (`worker_failed`) for a kept start failure + (`src/runtime/server.ts:271,297`), so it is final. A crash after start does not block today + (`src/runtime/server.ts:391-393`); with re-acquire, N crashes within T become sticky like a failed start. The owner's + Retry (`src/runtime/server.ts:475`) clears the failure and revives the ended references of that worker. +- **Only the origin re-acquires.** Relays (`connection.ts:183-190`) forward the loss and do not re-acquire on their own. +- **Bindings.** A remote reference tracks the channel plus the hold's incarnation: decoders are reset, subscribers kept, + late frames from the old incarnation ignored (`remote.ts:84-95`). `ready` belongs to one binding generation; UI hooks + subscribe to `onStatus`. Calls while re-acquiring wait up to a deadline (the worker's `readyTimeout`, 60 s, + `src/runtime/server.ts:430`), then fail. +- **References held by factories.** A re-acquired reference points to a new object: the holder's resources declared + on it (`deps`) and its listeners on it reopen. A changed variant or fingerprint after re-acquire is emitted as a shape + change. +- Calls in flight at the loss fail (as on a lost link) and may be retried. + +Statuses: `loading` (initially and while re-acquiring), `live`, `ended` with a reason (released by the holder, or a +final error). `released` and `gone` merge into `ended`. The reference object stays the same; state subscribers get a +fresh snapshot. + +**Idle stop vs re-acquire (decided: a).** A worker stops when idle; today only `Chat` and `WorkspaceBackend` count as +viewers (`plugins/chat/worker.ts:520`, `plugins/workspace/worker.ts:72`). Any other held worker object (TUI watches, +lingering holds) would re-acquire after an idle stop and restart the worker at once. Decision: any worker object held +by a client keeps the worker alive (holding is viewing); the per-plugin viewer counters go away. + +**Touches.** Chord: `remote.ts` (bindings, incarnations), `record.ts` (reopening dependents), `connection.ts` (reasons +across the wire, relays), `host.ts` (`invalidatePrincipal`). p2: `linger.ts`, `useObject`/`useStatus` in `web.tsx`, the +backoff in the TUI `watch` (`tui.ts`, lifted into Chord), the start failure reason and crash cap in +`src/runtime/server.ts`. + +## 7. API cleanup pass: declarations by name, void methods (batch with 1, 2, 3, 5) + +**Today.** Declarations are maps of name to a wrapped declaration: `self.state({ notes: cell(schema, initial) })`, +`self.derive({ count: derivation([sources], fn) })`, `self.resources({ db: resource({ rev, … }) })`, +`self.tasks({ load: fn })`, `self.listen({ view: listener(source, select, fn) })`. The spec chose property names in +object literals on purpose (`packages/chord/docs/old/spec.md:76`), with no recorded reason beyond that (presumably duplicate +names caught by TypeScript, several declarations per call). Methods without a result declare `Type.Null()` and return +`null`; methods without arguments declare `Type.Null()` and are called with `null`. The state version is per object +(`serve(…, { state: { version, migrate } })`). In p2: 28 `self.state` with 41 cells, 16 `self.derive` with 19 +derivations, 44 resources, 10 task blocks, 15 listeners; 62 `Type.Null()` results, 16 `Type.Null()` arguments, 90 +`return null;`, 10 `(null, ctx)` calls; state versions on two objects (status v2, subagents v3). + +**Rule.** A declaration has a name only if something of it survives a reload and must be found again: state (its +value) and resources (the open value). Derivations, tasks and listeners hold nothing across a reload. + +**Change.** + +```ts +const notes = self.state("notes", Type.Array(NoteSchema), []); // version 1 by default +const prefs = self.state("prefs", PrefsSchema, {}, { version: 2, migrate: (old, from) => … }); +const count = self.derive([notes], (n) => n.length); // no name +const db = self.resource("db", { version: 1, deps: [server], open: () => …, close: (db) => db.close() }); +self.task(async (ctx) => { … }); // no name +self.on("view", watch, watchSource, (event, ctx) => { … }); // name kept for now, see below +return { notes, count, add: … }; // exposed state = what is returned + +members: { add: method(Type.Object({ text: Type.String() })), reload: method() } // no result = void +add: async ({ text }, ctx) => { … }, // returns nothing +await store.reload(ctx); // no-argument call +``` + +- **State.** `self.state(name, schema, initial, { version?, migrate? })` returns the cell. On reload, the same name + returns the existing cell and its value (`initial` ignored); a raised version runs `migrate(old, from)` and validates + the result; a schema change without a version bump refuses the reload, so the version is optional (default 1). The + version moves from the object to the cell. Private state is state that is not returned; exposed state is returned + as members. A nested value stays one cell (`change` sends only the changed paths). +- **Derivations.** `self.derive(sources, fn)`: no name; recomputed at commit (today kept with the function replaced). +- **Resources.** `self.resource(name, { version, deps?, exclusive?, open, close? })`: `rev` becomes `version` + (required: a native value has no schema to detect a change); `deps` and `exclusive` keep today's semantics. The open + resource redesign (leases, lifetimes, Durable sources) is not part of this pass. +- **Tasks.** `self.task(fn)`: no name; restarted at every reload as today. +- **Listeners.** `self.on(name, source, [select,] handler)`: today's semantics (a kept subscription with the new + handler, matched by name), so the name stays for now. It goes away when listeners resubscribe at commit, which needs + the shared Durable sources from the base (the open listener redesign). +- **Methods.** `method(args?, result?)`: an omitted result is `void` (handlers return nothing), omitted arguments mean + a no-argument method called as `ref.name(ctx)`. Chord knows both from the contract on both sides; on the wire it + sends `null` internally. +- `cell()`, `derivation()`, `resource()`, `listener()` and the declaration maps go away. + +**Batch.** One pass after the in-place relocation lands (it touches the same plugin files), together with entries 1 +(terminology, `variant` with a link protocol version), 2 (object types declare their host), 3 (typed principals), +5 (`lifetime`), 8 (an awaited reference has its state) and 9 (state `init`). Chord first, via pi (Chord source, tests, docs `spec.md`/`guide.md`/`tutorial.md`; CI), then p2. Not +in the batch: 4 (`files` tool), 6 (references re-acquire), the resource and listener redesign. + +## 8. An awaited reference has its state, local or remote (batch) + +**Today.** For a local reference, `await ref.ready` means the object is live and its state exists. For a remote one it +only means the object is live (`shape`). Remote state is transferred only while someone listens: snapshot, then deltas. +A remote readable unsubscribes upstream and forgets its value when its last listener leaves (`remote.ts:81-86`). So +`ref.notes.value` after `await ref.ready` can be `undefined`, and state members are typed `T | undefined` +(`contract.ts:258,274`). Code asserts the value where it is guaranteed (in `open` with `deps`, in methods, listeners and +tasks): p2 has 27 `.value!` in plugin and runtime code (33 with tests) and 98 `!` on opaque members. A resource's +`.value` throws `not_ready` instead (`record.ts:121-125`, in a factory `:191`). + +- Every remote `get` makes its own remote reference and its own hold over the link (`#acquireRemote`, `host.ts:811`), + each subscribed member its own feed. Every hold frame carries the full contract description (`connection.ts:466`, + `wire.ts:68`). p2 shares holds by hand: `linger.ts` (keyed by type, input and `at`; 2 min, at most 4, `keep` + subscriptions) for some hooks and TUI watches. +- An object already goes live only after its declared references are live (`record.ts:923`; reload prepare + `reload.ts:372-376`). +- The provider computes the diff of a change once and fans it out (`cell.ts:34-49`). Each feed then encodes it with its + own stateful encoder (`connection.ts:812`), and each link serializes it and deflates it above 64 KiB + (`link.ts:186`). +- A provider skips a snapshot whose value is `undefined` (`connection.ts:820`). State waits behind the ack window, one + slot per subscription in creation order, up to 4 windows each (`connection.ts:26,363,402`). + +**Change.** After `await` on a reference, every state member of its variant has a value, local or remote. + +- **Ready.** A reference is ready when its object is live and every state member of its variant has its first snapshot. + This includes state a local object forwards from, or adapts from, a remote reference it holds. +- **Gets.** `ctx.get` and `host.get` are async: they resolve once the reference is ready, and reject with the activation + error (`forbidden`, unreachable, …). `self.get` stays synchronous: the object goes live only after its declared + references are ready and hydrated. `ready` disappears from plugin code. +- **Awaiting.** An awaited get races `ctx.signal` and ctx close; `host.get` takes an `AbortSignal`. The deadline is the + worker's `readyTimeout` (60 s, as in entry 6). Rejection or abort releases that view only. A pending await counts as a + reference, so one waiter giving up does not tear down a binding others wait for. +- **One binding per host and key.** The key is (`at`, type, normalized input, principal key, contract fingerprint). + Today's remote address uses the raw input and no fingerprint (`host.ts:993`). During a reload, old-epoch and + candidate code hold two fingerprints, so two bindings exist until retire. + - All references with the same key are counted views onto one binding: one hold, one feed per state member, calls over + the same binding. + - Views are distinct proxies with their own owner lease (factory, ctx, host, client), their own `release` and their + own observers. Gone, re-acquire (entry 6) and ending by a final error belong to the binding. + - Relays stay 1:1 per origin hold. The browser's binding already collapses its components into one relayed hold. + Different principals and different hosts (two tabs) stay separate. +- **Wire.** A hold implies a subscription to the state members of its variant. The provider sends the shape and the + first snapshots as one unit: a new link protocol version (entry 1). No `sub` frames in the normal case; `sub` and + `unsub` remain only for the opt-out. + - `undefined` is sent as an explicit snapshot, so it hydrates. + - The first snapshots of a binding that is not yet ready go past the ack window: a get does not wait behind other + bindings' slots. Call results keep their priority. + - A per-connection cap bounds pending state. Beyond it, the oldest slots are replaced by resets. + - The diff is computed once per change (as today); encoding and serialization stay once per feed (stateful + per-subscription encoders). +- **Opt-out.** `get(…, { state: false })` for method-only views: the workspace panel's `Chat`, TUI commands. Such a view + is ready once live. The binding subscribes if any of its views wants state. +- **Variants.** A shape change is applied only once every new state member has its snapshot. A member that is not in + the current variant throws `not_in_variant`. References are typed as a union discriminated by `variant` (entry 1): + another variant's members are not accessible without narrowing. +- **Status.** State members are typed `T`. `not_ready` is thrown only before a binding's first hydration. While the + binding re-acquires (entry 6) the last values stay readable with status `loading`. Reading an ended view throws. + Opaque members of a live local object are typed without `undefined` the same way. +- **Linger.** A consumer-side linger on the binding replaces `linger.ts` and its `keep`: a per-type option and a + per-host cap (p2: 2 min, at most 4). The provider-side `linger` (`record.ts:1234`) stays. +- **UI.** `useObject` (`web.tsx:140`) and the TUI `watch` (`tui.ts:84`) keep a synchronous internal hold (`host.hold()`) + and render loading from status. `useReadable` (`web.tsx:201`) gates on status, so nothing throws during render. +- **Startup and cycles.** Worker code must not await state-bearing remote references before `control.ready()` + (`src/hosts/worker.ts:61-88`). Today's worker-to-server references are caller-only (`SessionStatus` and `Subagents` + `reporter`). Activation and reload prepare get a ready deadline that fails with a cycle hint instead of hanging + (cross-host cycles are not detected, `host.ts:1000`). +- **Cost.** A reference held only for calls receives no state when it uses the opt-out. Objects held by several + components (`Access` 7×, `Sessions` 6×, `Subagents` 3×, `Chat` 2×) cost one copy per client. + +**Touches.** Chord: `remote.ts` (binding, views), `host.ts` (key, async `get`, `hold`), `connection.ts` and `wire.ts` +(hold with state, shape plus snapshots, explicit `undefined`, first-snapshot priority, per-connection cap, protocol +version), `record.ts` (hydrated readiness, forwarded state, deadline, `get` in `SelfImpl` and `CtxImpl`), `contract.ts` +(typing). p2: `linger.ts` (removed), `useObject`/`useReadable`/`useStatus` in `web.tsx`, `watch` in `tui.ts`, every +`.value!` and `await ref.ready` in plugins. + +**Tests** (a counting link, frames and bytes per direction: the `AckGate` fixture in +`packages/chord/test/old/flow.test.ts`; faux timers): + +- **Traffic.** N views (factory, `ctx.get`, `host.get`, hook) on one key give one hold, one first snapshot per member + and one delta frame per change. Releasing N−1 views sends nothing. Releasing the last sends `release` after the + linger, not before. A new view within the linger sends nothing. Hooks mounting and unmounting send no frames. +- **Identity.** One binding for normalized-equal inputs and equal principal keys. Separate bindings for different + principals, grant revisions and fingerprints. Two fingerprints during a reload, one after retire. +- **Readiness matrix.** Local state, forwarded remote state, derivations over remote state, `null`, empty and + `undefined` values, state-free variants. A failed snapshot (validation error) rejects the get instead of hanging. +- **Variants.** On a variant or contract change at reload, no moment exists where the view is live and a member read + throws. A removed member throws `not_in_variant`. New members arrive with the shape. +- **Awaiting.** Abort, ctx close and the deadline (link down) release that view only. The binding survives for other + waiters. The root ends, so a reload gate drains. +- **Re-acquire and reconnect.** Exactly one snapshot per (binding, member). Frames from the old incarnation are + ignored and no subscriptions are duplicated. A view that joins during a re-acquire waits for the fresh snapshot. The + last values stay readable meanwhile. +- **Opt-out.** A `state: false` view alone transfers no state. When a full view joins, the subscription starts once. + The final release unsubscribes. +- **Relay topology.** One origin hold gives one downstream hold and one feed per member, with nothing left after + release, link close or revoke. Two tabs, different principals. A stalled tab overflows and resets while another tab + keeps getting deltas. +- **Flow and load.** + - Delayed acks with three 500 KiB snapshots queued: a new get of a small object resolves after about one round trip, + and call results are not delayed. + - A chat snapshot larger than the window. + - IPC (no sizes) between a worker and the server. + - A stalled peer with K subscriptions stays under the per-connection cap, then gets one reset per feed. + - Many clients on one object: server memory per client is bounded. +- **Provider cost.** One diff per change and K encodes for K feeds (spy on `encoder()`), and the deflates per snapshot. +- **Startup and cycles.** A worker with `awaitEager: false` and eager objects holding server objects goes live after + `control.ready()`, with one snapshot each. A cross-host hold cycle fails at the deadline with a cycle hint. +- **Types.** Another variant's member is not accessible without narrowing. A local `self.get` member is typed `T`, + reads without `!` once live, and throws `not_ready` inside the factory. + +## 9. State initialized from its source before the object goes live (batch) + +**Today.** Stores load their state in a task: `self.tasks({ load: async (ctx) => state.replace(ctx, db.all()) })` +(catalog, subagents, ports, sharing). Tasks start once the object is live, so calls and watchers can see the initial +value (`[]`) first: watchers converge, but a method reading the state in that window answers wrong (a note that exists +reads as unknown), and an asynchronous load (Durable) can overwrite a write made during its `await` with a stale +snapshot. The chat-mode mirror works around this with a reread loop. + +**Change.** `self.state(name, schema, initial | { init })`: `init` (may be async) runs after the object's resources +are open and before it goes live, so no call or watcher sees a value that is not from the source. On reload `init` does +not run (the state survives). The `load` tasks go away. Superseded by target-design §7: every cell has `init` (no fixed initial +value); on a bump, a pure `migrate` converts the old value to the new schema and `init({ previous })` gets the result. + +## 10. `lifetime: "host"` checked by the type system + +`serve(Type, factory, { lifetime: "host" })` compiles only if `Type` is local and its input has no required fields (no +new method). Today Chord checks this at runtime (`invalid`). + +## 11. The facet is an implicit host-lifetime object + +**Today.** A facet's setup (`defineFacet((host) => …)`) serves types and reads registries, but cannot register, hold +references, state or resources. Contributions need a dummy host-lifetime object (`contribute(host, name, self => …)`, +47 in p2), and sharing a native thing within a plugin needs an object type with an `opaque` member and `expose` +(`TerminalServiceObject`, `DeviceAccess`, radius, codemode). + +**Change.** `defineFacet((host, self) => { … })`: the facet's setup is the factory of an anonymous object with lifetime +`"host"` that nobody else can reach. + +- It declares state, resources, effects and registrations directly, and serves the plugin's types. +- Objects of the plugin's types share the facet's declarations through the closure: no extra type, no `expose`, no + `opaque` within a plugin. Between plugins: contracts and registries only. +- Registering is allowed from the facet object and from explicit host-lifetime objects; never from referenced objects + (their entries would come and go with references). +- The setup follows the factory rules: it only declares, runs again on every reload (first as a trial), its local + variables reset; whatever must survive is `self.state`. +- Behaviour built on a shared native thing is an effect (restarted with the new code). A kept resource built on a facet + resource names it in `deps` (allowed across the facet object and its plugin's objects); replacing the facet resource + replaces it in every object of the facet. +- `contribute` and the dummy types go away. + +## 12. The effect model + +| Survives a reload? | Declaration | +| --- | --- | +| yes, data | `self.state(name, schema, initial \| { init }, { version?, migrate? })` | +| yes, a native value (DB handle, SSH connection, sandbox, terminal host) | `self.resource(name, { version, deps? }, open, close)`: always named and versioned, holds only the native value, no behaviour | +| no | `self.effect(setup)`: `setup({ signal, run })` starts behaviour and returns its cleanup; restarted with the new code on every reload (the old one stops first) | +| no, computed | `self.derive(sources, fn)` | + +- `run(fn)` starts a call tree with a `ctx` per event, so a handler can write state and call other objects. +- Listeners, tasks and timers are effects; `onEvents(source, handler)`, `every(ms, fn)`, `loop(fn)` are plain helper + functions returning an effect setup, not Chord concepts. +- Gone: `self.on` (names, `select`, source-text matching, kept subscriptions), `self.task`, unnamed resources. +- Rule: resources hold only native values; behaviour goes into effects. Stale code is only possible in kept resources. +- Needs shared Durable sources in the base (one watch per conversation, fanned out), since Durable watches allow one + subscriber and effects resubscribe. + +## 13. Versioned registries + +Registries get versions and adapters like contracts, without a structural check (entries are code and native objects, +not data). + +```ts +export const Routes = defineRegistry("p2.routes", { + version: 2, + adapters: { 1: (old: RouteHandlerV1): RouteHandlerV2 => (req, res, access, ctx) => old(req, res, toV1(access), ctx) }, +}); +``` + +- The definition lives in the owner's versioned contract package (p2's or a plugin's: `SendRouters`, + `TranscriptEntries`, `SessionArchived`); a plugin's registration carries the version it was written against. +- At registration: same version → accepted; older with an adapter → wrapped; otherwise refused at load with a clear + error (also for a registry name the host doesn't have). No silent dead entries. +- No automatic coverage check: the owner bumps the version for breaking changes and writes the adapter. + +## Open (not decided) + +Discussed in the walkthrough, not decided; each needs a decision before it is built. + +- **Native values from the host for facets:** host values (plain, ready before any facet) instead of runtime objects + with `opaque` members (`Harness`, `WorkerProcess`, `Server`, `HostLifecycle`); no `expose`/`opaque` for plugins + across plugins; the harness opens without plugin waits. Natives can't be adapted, so the native surface stays minimal. +- **Viewer counting and presence:** today enter/leave resources (sharing presence, status viewers, login abandon, worker + liveness viewers). Proposed: stores count per-lease tokens (`enter` returns a token, `leave(token)`), so reload order + does not matter. Dynamic registries were rejected (a hack). +- **Removing the gate:** a changed state version or kept resource restarts the object instead of migrating under a gate. + +## Tracked, not scheduled + +Work found along the way, parked until the object-state redesign (`target-design.md`) is decided. Do not lose. + +- **Leaks** (`plugin-loading.md` 14.4): C3 still open in Chord (`object.ts` `stageListeners`: a kept listener + subscription keeps the previous run's `select`; fix: adopt `decl.select` on the kept path). C5 (p2 + `PluginRuntime.guard` adds a listener per reload) and C7 (Harness `execution` keeps the envs facet's `close`) open. + C6 belongs to loader D. C1, C2, C4 fixed in `00d8d0495`; C10 gone with entry 8. Rerun the generation-GC tests after + the redesign. +- **p2 behaviour on interrupted calls** (fence-impact assessment): Chat shell sampling timer writes a cell from a bare + `setTimeout` (an error there exits the worker); `admitted` shell dedupe lives in a factory closure (lost on reload); + `stop`/`dequeue`/`claim` return withdrawn text only in the reply (lost if the reply is lost); web Send does nothing + while disconnected (TUI queues); new session plus send and review comments have no request id (duplicates on retry); + terminal keystrokes are dropped while disconnected; sharing revocation loop and `relocate`/`clone` are multi-step. +- **Durable:** nested commits hang by convention only (documented, not detected); `AsyncLocalStorage` would be needed + to detect it. No change planned. +- **#10959** (remote references accept a foreign ctx): the owner fixes it with a subagent. + diff --git a/apps/p2/docs/chord-next-walkthrough.md b/apps/p2/docs/chord-next-walkthrough.md new file mode 100644 index 000000000..c444690b6 --- /dev/null +++ b/apps/p2/docs/chord-next-walkthrough.md @@ -0,0 +1,1839 @@ +# Chord v2 and its plugins: a guided walkthrough + +A step-by-step introduction to how plugins work on Chord v2 (`packages/chord/src`, used by the playground and +p3), for someone who knows nothing about either. It teaches the **design**: the target design +`packages/chord/docs/design.md` together with the decision log `packages/chord/docs/decisions.md` (the log is +newer and wins where they differ). It does not describe the state of the code. API names are the design's working +names; where the kernel already has a name (`packages/chord/src`), the examples use it. + +Points the design has not settled are marked **Open** and list their options. Nothing in an Open block is decided. + +--- + +## For the agent guiding the user + +You walk a user through this document. Follow these rules: + +1. **One concept at a time.** Explain it in your own words, in the user's language, with the document's example. + Keep it concrete; don't paste whole sections. Explain in full sentences in this document's tone, not as terse + notes; define every word the user may not know when it first appears (e.g. IPC, relay). +2. **Then ask the check questions** of that concept and **wait** for the answers. +3. **Judge the answers** against the model answers (they are for you). If an answer is wrong or half right, say + precisely what is missing, explain that part differently, and ask again. Don't move on while a concept is unclear. +4. **Counter-questions come first.** Answer a user's question completely before anything else. If the answer lies in + a later concept, give a short answer and say where it continues. +5. **Move on only when the user says so.** +6. **No terms before they are introduced.** If you need one, introduce it first. +7. **Teach the design, not the implementation status.** For questions beyond this document, answer from, in this + order: `redesign-decisions.md` (newest decisions, wins on conflicts), `packages/chord/docs/design.md` (the + target design), `packages/chord/docs/kernel.md` and `step-*.md` (names the kernel already uses), + `plugin-loading.md` (loading, project plugins), `questions.md` (the Questions service), + `research/chord-on-durable-objects.md`. `target-design.md` and `p3-walkthrough.md` are superseded. +8. **Open is open.** Present every Open point with its options; never present one option, or your own idea, as + decided. If a question has no answer in the sources, say so plainly. +9. If you find a gap or a contradiction in the sources while teaching, tell the user and note it for whoever + maintains the design. +10. Keep a short list of what the user found hard; come back to it at the end. + +Start by telling the user the note below, then begin with "The setting". The setting has no check questions: ask +whether the user has questions, then move on when they say so. + +## For the user + +**Keep asking until you really understand a thing.** There are no silly questions. The concepts build on each other; a +gap early on makes everything after it harder. If an example doesn't click, ask for another one. If a word is unclear, +ask what it means. + +--- + +## The setting and the running example + +**Chord** is the runtime under an app: it lets plugins define **objects** that live in one place and lets every part +of the app use them, also across process boundaries. + +An app on Chord runs as several participants. Chord calls each participant a **host**. In a Node app (the playground, +p3) there are three roles: + +``` +browser browser TUI clients: any number, not addressable + └─────────┼───────┘ + server exactly one + ┌─────────┼─────────┐ +worker S1 worker S2 worker S3 one per session, started on demand +``` + +- **The server** (exactly one) serves the UI, holds data shared by all sessions and starts the workers. +- **A worker** (one per session) runs the agent and works on that session's files. The server starts it as a child + process when the session is first needed. +- **Clients** are browser tabs and TUIs (terminal UIs). They connect by themselves; nobody can send a request *to* a + client, so clients are not addressable. A client's own objects are therefore only ones nobody outside it needs + (concept 6 names them: `local` types). + +**How they connect.** Each connection between two hosts is a **link**: + +- Clients connect only to the server, over a WebSocket. +- The server talks to each worker over **IPC** (inter-process communication: the pipe between a parent process and the + child it started). +- Clients never talk to a worker directly; the server **relays** (passes requests on). A browser call to something in + session S1 travels browser → server → worker S1, and the answer comes back the same way. Workers reach each other + through the server too. +- When a client connects, the server checks who it is (for example its cookie) and attaches that to the whole link. + Concept 3 builds on this. + +**The goal that shapes everything: reload.** You change a plugin's code and the new code takes over inside the +running app. Two words matter: + +- a **reload** swaps a plugin's code while the hosts keep running: data, open connections and running work survive + where they can; +- a **restart** ends a process and starts it again: everything it held in memory is gone. + +Reloads are always explicit (a button or a command), never on save. Changes to Chord itself or to the app's runtime +are **base code**: they restart the hosts. Much of the design exists to make reloads safe; concept 23 shows the reload +step by step. + +**Base code and plugins.** Whether something is "core" or a plugin is decided by where its code lives: + +- **base code** (the app's `src/`): imported directly by the start code under a fixed name such as `@app/core`; always + there, not switchable; a change restarts the hosts. Core is what the app always needs and everything builds on (the + agent harness, the session list, access, the session location). +- **a plugin** (its own directory): found by listing, loaded by the plugin loader, switchable; a change reloads it. + Optional features are plugins. + +Both use exactly the same building blocks; base code is not special to Chord. + +**A plugin** is a folder with a `package.json`. Its `name` (globally unique and stable, like an npm package name, e.g. +`@playground/notes`) is the plugin's identity: Chord identifies the plugin's facets by it (their stored documents, +reloads, error messages), never by a directory name or load order. Base code's core facet gets a fixed name the same +way (`@app/core`). Each other file is meant for certain hosts; the part of a plugin for one kind of host is a +**facet**: + +| File | Runs on | Holds | +| --- | --- | --- | +| `package.json` | (read by the loader) | the plugin's `name` | +| `contract.ts` | every host that imports it | descriptions only: names, shapes, method signatures | +| `server.ts` | the server | the server facet | +| `worker.ts` | every session worker | the worker facet | +| `web.tsx` | the browser | the web facet (UI) | +| `tui.ts` | the TUI | the TUI facet | + +**The running example: a notes plugin.** The whole walkthrough builds this one plugin, piece by piece: + +- notes per session, kept across restarts; +- a web panel shows the notes; the owner and guests with write access can add and remove notes; read-only guests can + only read, and don't see who wrote a note; +- a text file dropped into the session's `inbox/` folder becomes a note; +- notes older than 30 days are deleted every hour; +- importing a large file runs in the background and can be cancelled; +- `GET /s/:session/notes.txt` downloads all notes; +- the panel shows how many people are looking at the notes. + +--- + +## Concept 1: Object types and objects + +**The problem.** Several places need "the notes of session S1": the owner's tab, a guest's tab, the download route. If +each built its own copy, the copies would drift apart, and nobody would know when to clean them up. + +**The idea.** There is **one object** for "the notes of S1". Everyone asks Chord for it by its **address** and gets +that same object. + +- An **object type** describes what such an object is: its name, what its address looks like (the **input**), where + it lives, what you can do with it (its **members**: data you can read and methods you can call) and a version. It + contains no code that runs; every host can share it. Another word for it is **contract**. +- A **factory** is the code that builds an object of that type, on the host where it lives. +- An **object** is one concrete instance: "Notes for `{ session: "S1" }`". + +The three pieces live in three files of the notes plugin: + +```ts +// contract.ts: the object type +export const Notes = remote("notes", { + version: 1, + host: /* where it lives: concept 2 */, + input: Type.Object({ session: Type.String() }), // the address: { session: "S1" } (a TypeBox schema) + variants: { /* the members: concept 3 */ }, +}); +``` + +```ts +// worker.ts: the factory, on every session worker +export default defineFacet((host, self) => { + host.serve(Notes, (input, principal, self) => ({ /* members */ })); +}); +``` + +```ts +// web.tsx: a caller in the browser +const notes = await host.get(Notes, { session: "S1" }); // browser → server → worker S1 +``` + +Read `host.serve` as "this host builds `Notes` objects like this". `principal` comes in concept 3, `self` in concept 7. + +- **Who builds it:** the host where the object lives runs the factory. That is why the `Notes` factory is in + `worker.ts`. +- **One object per address:** the first `get` for `{ session: "S1" }` runs the factory; every later `get` for the same + address gets the same object. `{ session: "S2" }` is another object, on another worker. (Concept 3 refines this.) +- **Methods return data, never objects.** If a method wants to point you to an object, it returns its address (`{ + session: "S1" }`) and you `get` it yourself. + +**Check questions** + +1. Two tabs ask for the notes of S1. How often does the factory run? +2. What is the difference between `Notes` and "Notes for S1"? +3. Why does the factory live in `worker.ts`, not in `server.ts`? + +*Model answers:* (1) Once: same address, same object (concept 3 refines this when the callers are different people). +(2) `Notes` is the description (the type); "Notes for S1" is one instance (an object). (3) Notes of a session live on +that session's worker, and the host where an object lives runs its factory. + +--- + +## Concept 2: Where objects live: host kinds and placement + +**The problem.** A caller asks for "Notes for S1". Chord must find the host that holds it, without the caller knowing +host names, and the worker must be sure the object really belongs to it. + +**The idea.** The app describes its addressable hosts as typed **host kinds**; one concrete host of a kind is an +**address**. A type's `host` field is a **placement function**: input in, address out. + +```ts +// the app's runtime (base code, not a plugin) +export const Server = hostKind("server"); // one host +export const Session = hostKind("session", Type.Object({ id: Type.String() })); // one host per id + +Session({ id: "S1" }) // an address: the worker of session S1 +Server() // the address of the server +``` + +```ts +// notes/contract.ts +export const Notes = remote("notes", { + version: 1, + host: ({ session }) => Session({ id: session }), // typed: `session` must exist in the input + input: Type.Object({ session: Type.String() }), + variants: { … }, +}); +``` + +**A call, step by step:** + +``` +browser get(Notes, { session: "S1" }) + runs the placement function → Session({ id: "S1" }) + has only a link to the server → sends the request there, with the address +server the address is not mine; the app told me how to reach Session hosts (concept 4) + → link to worker S1 (started if needed), forward +worker S1 runs the placement function too → Session({ id: "S1" }) = me → runs the factory +``` + +- **The caller computes, the provider checks.** The caller runs the function on the normalized input and sends the + result. The provider runs it too and refuses an object that is not its own with the error `invalid/placement`; a + worker for S2 never builds "Notes for S1". (During a rollout, concept 24, such a refusal is handled like a mismatch: + a provider on another revision makes the call park; on the same revision it fails.) +- **No host-name strings** in app code. Addresses are typed values; Chord serializes them internally. +- **Placement is not part of the type's fingerprint** (the hash Chord uses to compare two versions of a type, concept + 26). If the rule changes between versions, the provider's refusal surfaces it. +- A function is type-checked: a `Session` placement on a type without a `session` field does not compile. + +**Check questions** + +1. A tab asks for `Notes { session: "S2" }`. Which host builds it, and who computed that? +2. Why does the worker run the placement function again, if the caller already did? +3. What does app code never write when it names a host? + +*Model answers:* (1) The worker `Session({ id: "S2" })`; the caller computed the address with the type's placement +function, the server routed by it. (2) To refuse objects that are not its own (a wrong or outdated rule at the caller +must not create an object in the wrong place). (3) A host-name string; it calls the host kind (`Session({ id })`). + +--- + +## Concept 3: Who is calling: principals and variants + +**The problem.** The owner may add and remove notes; a read-only guest may only read and should not even see who wrote +what. The object must know who calls, and the caller must not be able to lie about it. + +**The idea.** Every request carries a **principal**: who calls. The app defines its principal type; every variant has +a `kind`. p3's principals: + +| `kind` | Who | +| --- | --- | +| `owner` | the person running the app (`access: "local" \| "relay" \| "device"`; a paired device has its `device` id) | +| `guest` | someone invited into one session (`session`, `mode`, its `grant` and the grant's revision) | +| `server` | the server calling a worker | +| `worker` | a session worker calling the server (`session`) | + +- **Links stamp principals.** A principal is never taken from a request's data: the server authenticates a client + when it connects and stamps that principal on the whole link; the server tells Chord who each worker is when it + connects it (concept 4). A caller cannot claim to be someone else. + +**Where principals come from.** Chord never sees cookies, share links or grants; that is app code (in p3 the sharing +feature, `p3-functions/sessions-access.md`). Chord only receives the finished principal and stamps it on the link: + +``` +owner: "copy share link" for S1, read/write + sharing (app code on the server) creates a link token for (S1, write) +guest opens the link → join page → enters a name + sharing stores a guest record (grant g1) and sets a guest cookie +guest's tab connects (WebSocket) + entry point (app code): authenticate(req): cookie → guest record g1 + → { kind: "guest", session: "S1", mode: "write", grant: "g1", revision: 1 } + server.accept(link, { principal }) ← Chord: stamped on the whole link +``` + +A **grant** is the app's stored record "this person may enter S1 with this mode"; `authenticate` checks it on every new +connection. + +**What principals do in Chord** (all of them pick something): + +- **identity:** they decide which object a caller gets (next point); +- **variant:** the factory receives the principal and picks what the caller may see (below); +- **admission:** `route.authorize(principal, address)` decides whether a caller may reach a worker at all (concept 4); +- inside methods `ctx.principal` says who calls (concept 12). + +**The caller is part of the object's identity.** One object exists per **(type, input, principal key)**. The +**principal key** is a string Chord computes from a principal: same key, same object; different key, different +object. By default it is the principal as canonical JSON (fixed field order). The app can supply its own function +(the `principalKey` option when it creates its hosts, concept 4) and so decide which fields split objects. An +illustration (the design does not fix p3's function): + +```ts +principalKey: (p) => + p.kind === "owner" ? `owner:${p.access}` : + p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` : + p.kind +``` + +| Caller | Principal key | `Notes {S1}` object | +| --- | --- | --- | +| owner, tab A, local | `owner:local` | 1 | +| owner, tab B, local | `owner:local` | 1 (shared) | +| owner via relay | `owner:relay` | 2 | +| guest g1, revision 1 | `guest:g1:r1` | 3 | +| guest g1 after a grant change, revision 2 | `guest:g1:r2` | 4 (new) | + +The design wants the key to include how the client is connected (access paths never share objects) and the grant's +revision (after a change from write to read the guest gets a new object, never the old `editor` one). + +**Variants:** the type lists named sets of members; the factory gets the principal and returns the variant that +principal may see. + +```ts +// contract.ts +variants: { + editor: { state: { notes: NotesSchema }, + methods: { add: method(AddArgs, NoteId), remove: method(RemoveArgs) } }, + reader: { state: { notes: PublicNotesSchema } }, // no authors, no methods +}, +``` + +```ts +// worker.ts +host.serve(Notes, ({ session }, principal, self) => { + if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; + return { variant: "reader", notes: … }; // filled in, concept 10 +}); +``` + +- **A reader has no `add` at all.** Chord refuses calls to members that are not in the caller's variant, so `add` + needs no check of its own. +- **The caller sees which variant it got** and narrows before using variant members; the type checker enforces it: + +```ts +const notes = await host.get(Notes, { session: "S1" }); +notes.add({ text: "hi" }); // type error: `add` exists only in "editor" +if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); +``` + +`expectVariant(ref, "editor")` asserts instead (throws `not_in_variant`). + +**The types know nothing about who calls.** Owner and guest run the same `web.tsx`; the type checker only forces every +caller to narrow before using variant members. Which variant a caller has is known only at runtime (`inVariant` is +`false` for the guest). The protection is Chord refusing the call on the provider; a malicious client that ignores the +types gets the same error. + +**Check questions** + +1. Why can't a guest become the owner by sending `{ kind: "owner" }` in a request? +2. The owner has two tabs open; two guests are in S1. How many `Notes` objects exist? +3. A read-only guest calls `add` anyway. What happens, and where was that decided? + +*Model answers:* (1) The link stamps the principal from the server's authentication; request data never sets it. (2) +Three: one for the owner (both tabs share it), one per guest. (3) Chord refuses it at runtime and the caller gets +`not_in_variant`: the guest got the `reader` variant, which has no `add`; the factory decided when it picked the +variant. (Not "it doesn't compile": the guest's code is the same as the owner's; types only force narrowing.) + +--- + +## Concept 4: Starting hosts: the low-level host API + +**The problem.** Someone must create the hosts, connect them, authenticate clients and start workers on demand. These +things differ between deployments (a loopback WebSocket, a relay, a test in one process), so Chord must not hard-code +them. + +**The idea.** Hosts are created by the app's own startup code (base code). Chord offers a small API; starting +processes, authentication, init data and keep-alive are app code. + +```ts +// app/server.ts +const server = createHost

({ + address: Server(), principal: { kind: "server" }, + storage: serverStorage, // a DocumentStorage (concept 13); in memory if omitted +}); +await server.load([ // facets with their plugin names + { name: "@app/core", facet: coreServerFacet(config) }, // base code's core facet + { name: "@playground/notes", facet: notesServer }, // a plugin +]); + +wss.on("connection", (ws, req) => { // an entry point: app code + const principal = authenticate(req); // cookie → owner or guest + if (!principal) return ws.close(4401); + server.accept(webSocketLink(ws), { principal }); // requests on this link act as `principal` +}); + +server.route(Session, { // how to reach hosts of kind Session + authorize: (p, { id }) => p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), + resolve: async ({ id }, { signal }) => { + const principal: P = { kind: "worker", session: id }; + const child = fork("worker.ts", [], { serialization: "advanced", signal }); + child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message + return { link: ipcLink(child), principal }; // Chord stamps `principal` on that link + }, +}); +``` + +```ts +// app/worker.ts +const { id, principal, init } = await startMessage(process); +const worker = createHost

({ address: Session({ id }), principal, storage: storageFor(id) }); +await worker.load([ + { name: "@app/core", facet: coreWorkerFacet(init) }, // serves the start data (concept 5) + { name: "@playground/notes", facet: notesWorker }, +]); +worker.connect(ipcLink(process), { principal: { kind: "server" } }); // the parent, and who it is +``` + +```ts +// a browser tab +const client = createHost

(); // no address: a client +await client.load([{ name: "@playground/notes", facet: notesWeb }]); +client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); +const notes = await client.get(Notes, { session: "S1" }); +``` + +| Call | Meaning | +| --- | --- | +| `createHost

({ address?, principal?, storage?, ... })` | with an address: addressable, its own code acts as `principal`; without: a client. `P` is the app's principal type; `storage` holds the host's persisted documents (concept 13; in memory by default) | +| `load([{ name, facet }])` | the facets for this host, already imported, each with its plugin's `package.json` name (a loader reads it; a test or static setup passes it; base code passes its core facet the same way); duplicate or invalid names are refused | +| `connect(link, { principal })` | the link to the parent (at most one; replaced after a reconnect); `principal` says who the parent is | +| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | +| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind: `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | + +**The first `get` for S1, step by step:** + +``` +browser Notes {S1} → Session({ id: "S1" }) → over its link to the server +server not me; route(Session): authorize(owner, { id: "S1" }) ok + no link to S1 yet → resolve({ id: "S1" }): fork, start message, IPC link + stamp { kind: "worker", session: "S1" } on that link → forward +worker reads its start message, createHost(Session({ id: "S1" })), load, connect + → serves the request: runs the Notes factory as the owner asked +``` + +- **`resolve` runs once per address while its link lives,** and again after the link drops (the worker crashed or + exited). While references to that worker still exist, Chord re-resolves by itself (with the per-address backoff of + concept 7), so a tab that only watches recovers without anyone clicking. A worker start that already produced a link + finishes even if the request that triggered it went away. +- **Incarnations:** a host is a logical identity (its address) plus an **incarnation**, a fresh id Chord mints on + every start and sends in the link handshake. The other side sees a restart as a new incarnation of the same identity, + and references to it re-hold (concept 8). +- **Entry points vary:** a loopback or tunnel WebSocket server, the Radius relay delivering channels, a Durable Object + edge, a memory link in a test. So Chord has no `listen`: each entry point authenticates and calls `accept`. +- **Tests** run everything in one process: `resolve` creates the worker host in memory and returns one end of + `memoryLinkPair()`. +- **Links** only deliver frames, in order, both ways. Heartbeats and recovery are the adapter's policy: IPC needs + none; a WebSocket through a relay needs a heartbeat (TCP does not notice a sleeping laptop). +- `host.root(fn)` gives code outside objects (a CLI command, a Durable hook) a call scope (concept 12). +- **Worker keep-alive: the worker counts, the app decides** (as in p2). Core code on the worker reports viewers + (concept 21), active calls and running jobs (Chord's counters), Durable tasks and objects kept warm by `linger` + (concept 7); the server's supervisor (app code) stops the worker when all are zero for the idle time (p2: 5 + minutes). Chord provides the counts, no idle logic. +- **Other options** of `createHost`: `principalKey` (concept 3; the same function on every host), `revokedBy` + (concept 11), `telemetry` (concept 29), `dev` (development checks), `onError` (background errors, including a + storage failure: Chord never ends the process, the app's start code decides; concept 13), and **timeouts**: host + defaults with per-thing overrides (closing a native value 30 s, activation and awaited `get` 60 s, parked calls 60 s, + routine handover 10 s, a rollout install 60 s). + +**Check questions** + +1. Who decides that the worker for S1 is a separate process, and where is that written? +2. The worker for S1 crashes. A tab calls `Notes {S1}` again. What happens? +3. Why does Chord have no `listen(port)`? + +*Model answers:* (1) The app: its `route(Session, { resolve })` forks the process; Chord only calls `resolve` when it +needs a link. (2) The link dropped, so the next request calls `resolve` again: a new worker starts, a new incarnation; +the tab's reference re-holds against it. (3) Entry points differ (loopback, tunnel, Radius, DO edge, memory); each +authenticates and calls `accept`. + +--- + +## Concept 5: Start data and core objects + +**The problem.** A worker's code needs things that exist before any plugin: the session's id, its init data, the data +directory. The start code got them in the server's start message (concept 4). How do plugins get them? + +**The idea.** There is no special mechanism. Start data is an ordinary object: a `local` type (usable only on its own +host, concept 6) with `lifetime: "host"` (it lives as long as the host, concept 7) and constant state, served by base +code's core facet from the start message: + +```ts +// base code (the app's src/): a local type and the core facet +export const SessionInfo = local("app.session-info", { + version: 1, input: Type.Object({}), members: { init: InitSchema }, +}); +export const coreWorkerFacet = (init: Init) => defineFacet((host) => { + host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); +}); + +// app/worker.ts (concept 4) +await worker.load([{ name: "@app/core", facet: coreWorkerFacet(init) }, …plugins]); + +// any plugin +const info = self.get(SessionInfo, {}); // a reference, declared in the factory +// later (in an opener, derive, routine or method): info.init.value.id, info.init.value.dataDir +``` + +- **Read after activation, never in the factory.** A factory only declares (concept 7); it gets a reference to + `SessionInfo` and reads `info.init.value` later: in the opener of a native value, a derive, a routine or a method. + Example: the inbox watcher (concept 14) builds its path from `dataDir` and the session id in its opener. +- **Paths are derived by the plugin** from generic start data (the data directory), never provided per plugin. +- **Platform bindings** the host owns (on a Durable Object its `state` and `env`) are core objects too: their + `self.open` (concept 14) returns the binding, their `close` does nothing. +- **A long-lived native value from the host** (the Durable harness, concept 28) is a `lifetime: "host"` object in the + core facet. +- Why no special "host value" mechanism: core objects already have everything (a type, a version, references), and + base code and plugins use the same primitives. + +**Check questions** + +1. How does a plugin on the worker learn its session id? +2. Why may a factory not read `info.init.value` directly? + +*Model answers:* (1) It declares `self.get(SessionInfo, {})` and reads `info.init.value.id` after activation (in an +opener, derive, routine or method); base code's core facet serves `SessionInfo` from the start message. (2) A factory +only declares; reading a reference's value happens after activation, when the reference is live and hydrated. + +--- + +## Concept 6: `remote` and `local` types + +**The problem.** A browser must reach `Notes` across links, with checked arguments and per-caller views. But two +objects on the same worker (the notes view and the store behind it) shouldn't pay for checks and principals. + +**The idea.** Two kinds of types: + +```ts +export const Notes = remote("notes", { version, host, input, variants }); +const NoteStore = local("notes.store", { version, input, members }); +``` + +| | `remote` | `local` | +| --- | --- | --- | +| reachable from | any host, across links | its own host only | +| identity | (type, input, principal key) | (type, input) | +| principals, variants | yes | no | +| factory | `(input, principal, self)` | `(input, self)` | +| arguments | validated once at the provider when the caller is not a host principal; passed by reference on the same host | neither validated nor copied | +| results | not validated in production; passed by reference on the same host | not validated, not copied | +| members | state and methods: data (JSON plus bytes) | plain-data state and methods | +| `ctx.principal` in methods | the caller | none | + +- **Validation by trust.** Arguments (and the input of a new reference) are checked against the **provider's** + TypeBox schema whenever the call originates from someone who is not a host principal: a guest, the owner, any + client, also when the server relays it over IPC. Calls from server and worker code itself are not validated in + production. Never with a schema a peer sent. The handler receives exactly its declared shape (undeclared fields are + stripped). Schemas are compiled once, at first use. +- **Values are immutable by convention, never copied.** On the same host a value is passed by reference, for both + kinds; nobody mutates a value it got. Development mode freezes values to catch mistakes. (Copies would cost time and + memory on every call; the convention is the same as in Durable.) +- **`local` is the trusted path:** no principals, no checks. Attribution ("who wrote this note") is passed as data, + because a local object has no principal. +- **Variants stay compatible:** a member name has the same kind (state or method) in every variant that has it, and a + method the same signature; only state schemas may differ per variant (the reader's redacted notes). +- **Clients serve only `local` types:** nobody can address a client, so nothing remote can live there. A client's + local calls have no principal; what a client calls on the server is stamped with the client's authenticated + principal (concept 3). +- **Members are data only,** on both kinds: no callbacks as arguments, no native values. Concept 19 explains why and + what to use instead. + +**Check questions** + +1. Why does `NoteStore`'s `add` take `{ text, by }` while `Notes`' `add` takes only `{ text }`? +2. Owner and guest both use `NoteStore {S1}`. How many `NoteStore` objects exist? + +*Model answers:* (1) A local object has no principal; the per-principal `Notes` view knows who calls and passes the +author as data. (2) One: a local identity is (type, input), with no principal. + +--- + +## Concept 7: Objects and their lifecycle + +**The problem.** An object needs references, data, maybe an open file watcher, before it can serve calls. If the +factory did all that itself (awaiting, opening), a reload would rerun it and open everything twice, and a failure +halfway would leave half-open things behind. + +**The idea: factories declare, Chord activates.** The factory is synchronous and only **declares** what the object +needs, through `self`; Chord performs it: + +```ts +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); // a document: concept 13 + const count = self.derive("count", [list], ([l]) => l.items.length); // concept 9 + const catalog = self.get(Catalog, {}); // a reference: concept 8 + self.routines.every("cleanup", HOUR, (ctx) => dropOld(ctx)); // concept 15 + const imports = self.job(ImportJob, { run, finish }); // concept 17 + return { notes: list, count, add: …, remove: … }; +}); +``` + +The lifecycle: + +``` +(first get) → activating → live → ending → ended + └→ failed (backoff) → activating … +``` + +- **Activating:** Chord binds the declared references (and waits until they are live), loads documents, opens native + values (concept 14), computes derives, starts routines. Calls wait behind the object's **gate** until it is **live**. +- **Waiting for dependencies:** an object whose references are not live yet stays `loading` and goes live as soon as + they are (at most 60 s, then it fails). So when a dependency recovers, its dependents recover at once. +- **Ending:** routines stop, the object's calls and jobs are aborted as their rules say, native values close in reverse + order, references are released. +- **Lifetime:** `"referenced"` (default) lives while someone holds a reference, plus an optional `linger` after the + last one (set by the type's provider: keeps a just-closed object warm); `"host"` lives as long as the host (only for + types without a required input, checked by the type system). A running job counts as a reference (concept 17). +- **Failed:** any broken part fails the **whole** object (a half-working object could break rules that rely on its + documents changing together). Chord tags the cause by where it failed: + + ```ts + cause: { kind: "transient"; error } // an open threw, a dependency was unreachable: automatic backoff + | { kind: "broken"; error } // factory, derive, document loading or migrate threw: no automatic retry + ``` + + **Transient** causes get a **backoff per address**: 1 s, doubling up to 60 s; concurrent gets share one attempt; + success resets it. **Broken** ones (a code bug, invalid data) would fail the same way again, so they are retried + only by a trigger. Triggers, for both: a plugin reload, `ref.retry()` (a Retry button), and wake signals the factory + declares with `self.retryOn(...readables)` (sources that live independently of the attempt, such as the facet's + documents). Only an opener may declare its own failure `broken` (`throw broken(error)`, e.g. invalid configuration). +- **Handles stay valid** for the object's whole life: nothing is re-fetched or "read at use". + +**Check questions** + +1. Why may a factory not `await` anything? +2. `NoteStore {S1}` fails to activate at 10:00:00. A tab asks again at 10:00:00.5. What happens? + +*Model answers:* (1) It only declares; async work happens in activation, calls, routines and jobs. A reload reruns the +factory, so side effects there would repeat, and a half-finished factory would leave things behind. (2) It depends on +the cause. Transient (an open threw): the get is inside the 1 s backoff, it gets the stored failure (`failed` with +`retryAt`), and the next attempt runs at 10:00:01 unless a trigger comes first. Broken (the factory threw): no +automatic retry at all; only a reload, `retry()` or a `retryOn` signal starts a new attempt. + +--- + +## Concept 8: References + +**The problem.** A consumer uses an object that may be on another host, may not exist yet, and may go away (link drop, +restart, failure). It needs one handle that keeps the object alive and tells it what is going on. + +**The idea.** `get` returns a **reference**. While held, it keeps its object alive. Awaited, it is **live and +hydrated**: every state member of its variant has a value. Three ways to get one, differing in when it is released: + +| Where | API | Released | +| --- | --- | --- | +| in a factory | `self.get(Type, input)` | when the object ends | +| in a call | `await ctx.get(Type, input)` | when the call ends | +| outside (UI, tests) | `await host.get(Type, input)` | `ref.release()` | + +`{ state: false }` binds for methods only (no values are sent). + +**State members are readables.** A **readable** is a value plus its ordered changes: `.value` is the current value, +`.subscribe(fn)` is called on every change. A reference's state members (`notes.notes`) are readables, on any host. + +**`ref.state` says what is going on,** as a tagged value; `ref.onState(fn)` gets each new one: + +```ts +| { kind: "loading" } // first hold, reconnecting, restarting +| { kind: "live" } +| { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: broken, no automatic retry +| { kind: "released" } +``` + +**What calls and values do:** + +| Situation | Calls | Values | +| --- | --- | --- | +| link drop | park until reconnect | readable, `loading` | +| object loading or restarting | park until live | readable, `loading` | +| rollout: provider ahead or installing (concept 24) | park until installed | readable | +| object `failed` | reject at once with the cause | readable, `failed` with `retryAt` | + +- **Parked calls** share one default deadline (60 s, overridable per reference: `get(…, { parkTimeout })`): after it + they fail `unavailable`. Otherwise a call ends only through its caller's signal; there is no per-call deadline. +- **A failed reference stays held:** for a transient cause Chord retries per the backoff; `ref.retry()` tries at once; + success goes `loading`, then `live`. +- **Calls that were running** on an instance that ends fail with an error and are not resent. +- **Errors look like disconnects.** No request ids, no special protocol: a consumer handles a failed call as it would + a dropped link (show the error, the user retries). +- **Authority checks** live in the per-principal view's methods (and in `route.authorize` before a host starts). There + is no async check before an object goes live. + +- **No optional references.** If the other side's plugin may be missing (radius uses ports' forwards, but ports is + optional), the dependency is turned around: the dependent owns a registry and the other plugin contributes to it + (ports contributes its forwards to `radius.forwards`; without ports the table is empty and radius works). Registries + come in concept 19. `get` has no `optional` flag. + +**Check questions** + +1. The worker restarts while a tab shows the notes. What does the tab see, and what happens to an `add` clicked + meanwhile? +2. When do you use `self.get` and when `ctx.get`? + +*Model answers:* (1) The values stay readable with `ref.state` `loading`; the `add` parks; when the new incarnation is +up, the reference re-holds, gets fresh values, goes `live`, and the parked call runs. (A call that was already running +on the old worker fails.) (2) `self.get` in a factory for something the object needs its whole life; `ctx.get` in a +method for something one call needs. + +--- + +## Concept 9: Derived values + +**The problem.** The reader variant needs the notes without authors; the panel shows a count. Storing copies would go +stale. + +**The idea.** `self.derive(name, sources, (values, previous) => value)` computes a value from readables (documents, +reference members, other derives), recomputes when a source changes, and sends viewers only the difference. + +```ts +const count = self.derive("count", [list], ([l]) => l.items.length); +``` + +- `previous` is the derive's last value, for incremental work. It is kept across reloads by the derive's **name**. It + is only a cache (empty after a restart), so a derive must be correct without it. +- A derive that throws fails its whole object (concept 7). +- **How the difference is found:** Chord compares the new value with the old one (Delta's `diffRevisions`). Parts that + are the **same objects** as last time cost one `===` each; parts that are new objects must be compared by content. + +**The footgun.** A derive that builds new objects for unchanged data makes every change cost the size of the whole +value, per view. Example: 2,000 notes, the reader's view strips authors: + +```ts +self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ text: n.text, at: n.at })) })); +// one note added: 2,001 new objects, all compared by content +``` + +**`mapShared(f)`** (from `@earendil-works/chord/delta`) keeps one result per source element: `f` runs only for new or +changed elements; unchanged ones return their previous result object. + +```ts +const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); +self.derive("public", [list], ([l]) => ({ items: l.items.map(strip) })); +// one note added: one new object, 2,000 identical references, the diff emits one insert +``` + +**Cookbook** (patterns with examples): select a field, look up by key, small aggregates (just compute; the value is +small); filter or map a list (`mapShared`); redaction (preferably separate documents per visibility, otherwise +`mapShared`, computed once in the shared store, concept 10); joins across two sources; a window over a large list +(keep its start stable so changes arrive as appends). + +**Check questions** + +1. Why does a derive have a name? +2. The reader view maps every note to a new object on every change. What does one added note cost, and how do you fix + it? + +*Model answers:* (1) So Chord recognizes it across reloads and can hand the new code its `previous` value. (2) The +whole list: every object is new, so the diff compares all of them by content, per view. Use `mapShared`: unchanged +notes keep their result objects, and the diff finds the one insert by identity. + +--- + +## Concept 10: Pattern: shared store, per-principal views + +**The problem.** Each principal has its own `Notes` object (concept 3). If each one loaded the notes, or computed the +reader's redaction itself, there would be one copy per caller again: exactly what concept 1 wanted to avoid. + +**The idea.** Heavy or shared parts live once in a `local` **shared store**; the per-principal `remote` views get it +and expose what their principal may see. Views of the same variant read **one** value from the store: the redaction is +computed once, not per guest. + +```ts +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); // concept 13 + const strip = mapShared((n: Note) => ({ text: n.text, at: n.at })); + const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(strip) })); // once, for all readers + return { notes: list, public: pub, add: …, remove: … }; +}); + +host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); // one per session + const by = authorOf(principal); + if (canWrite(principal, session)) return { + variant: "editor", + notes: store.notes, + add: ({ text }, ctx) => store.add({ text, by }, ctx), + remove: (args, ctx) => store.remove(args, ctx), + }; + return { variant: "reader", notes: store.public }; // the store's one redacted value +}); +``` + +``` +NoteStore {S1} local shared store: the notes and their redaction, once + ▲ self.get ▲ +Notes {S1} as owner Notes {S1} as read-only guest (and every other reader) + variant "editor" variant "reader": store.public +``` + +- **Authority is the view's;** the store is trusted and acts for the host. +- The same pattern holds a Durable watch, a file watcher, a sandbox: once in the store, never per principal. Any plugin + uses it with the same primitives; base code is not special. + +**Check questions** + +1. Owner and two read-only guests view S1. How many `Notes`, how many `NoteStore`, how many redacted lists? +2. Where is it checked that the guest may not add notes? + +*Model answers:* (1) Three `Notes` (one per principal key), one `NoteStore`, one redacted list (`public` in the store, +shared by both readers). (2) In the `Notes` factory, which gives the guest the `reader` variant without `add`; the store +does not check. + +--- + +## Concept 11: Revocation + +**The problem.** The owner removes guest g1 from S1, archives S1, or signs out a paired phone. The open tabs concerned +must lose access at once, also on the worker where their `Notes {S1}` objects live, and reconnecting must not bring the +old access back. + +**What we build on.** The app turns a cookie into a principal (`authenticate`, concept 3); Chord stamps it on the link; +every object on a host exists per principal key, and Chord keeps the principal next to each object (it went into the +factory); every link has its stamped principal. + +**The idea: labels.** The app defines, once in its runtime, which **labels** a principal carries. Revoking means +revoking one label; everyone carrying it is hit. + +```ts +type Revocation = { grant: string } | { device: string } | { session: string }; // the labels + +const revokedBy = (p: P): Revocation[] => // the labels principal p carries + p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] + : p.kind === "owner" && p.access === "device" ? [{ device: p.device }] + : []; + +const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); +// every worker is created with the same revokedBy +``` + +`revokedBy(p)` is not a yes/no question: it returns the list of labels `p` carries. + +| Principal | Labels | +| --- | --- | +| guest g1 of S1 | `{ grant: "g1" }`, `{ session: "S1" }` | +| guest g2 of S1 | `{ grant: "g2" }`, `{ session: "S1" }` | +| owner on the paired phone | `{ device: "phone" }` | +| owner, local | none | + +| Call | Hits | +| --- | --- | +| `revoke({ grant: "g1" })` (remove one guest) | g1 | +| `revoke({ session: "S1" })` (archive S1) | g1 and g2 | +| `revoke({ device: "phone" })` (sign out the phone) | the phone | + +**What `host.revoke(r)` does** with one label `r`: + +``` +for each live object and each link on this host: + r in revokedBy(its principal)? (compared as canonical JSON; never the host's own principal) + → end the object / close the link +then send r to every child host, which does the same with its own revokedBy +``` + +On worker S1, for `revoke({ grant: "g1" })`: + +| Object | Principal | `revokedBy(p)` | Hit? | +| --- | --- | --- | --- | +| `Notes {S1}` | owner, local | none | no | +| `Notes {S1}` | guest g1 | `grant g1`, `session S1` | **yes** | +| `Notes {S1}` | guest g2 | `grant g2`, `session S1` | no | +| `NoteStore {S1}` | none (local) | none | no | + +Only guest g1's `Notes` ends; ending it releases its reference to `NoteStore`, which keeps serving the others. Why a +label and not a yes/no function: revocation is an event that must travel from the server to the workers, and a label +is plain data that can be sent; a function could not. + +**Removing guest g1, step by step:** + +``` +owner removes guest g1 + sharing (app): marks grant g1 blocked in its stored guest records + server.revoke({ grant: "g1" }) +server the guest's WebSocket link carries grant g1 → close it; g1's objects on the server end + send { grant: "g1" } to the workers +worker S1 g1's Notes {S1} ends +guest tab link gone; reconnects → authenticate: grant g1 is blocked → refused +``` + +- **Chord persists nothing here.** `revoke` only cuts live access. Lasting denial lives in the app's data, which + `authenticate` checks on every new connection. After a server restart there is nothing live to revoke; every client + must authenticate again, and blocked grants are refused there. +- **No re-stamping.** A changed grant (mode write → read) is a revocation of the old grant: the link closes, the tab + reconnects and authenticates with the new grant (new revision, so a new principal key), and its references + reconnect by themselves (concept 8), now as a reader. +- **Work running in a shared store** on behalf of a principal (a queued question, a job a guest started) is not ended + by `revoke`: the store has no principal (concept 10). Such a store keeps the grant as data with the work and + reconciles from the grant owner's live grants: a routine (concept 15) aborts jobs and drops queued input whose grant + no longer exists. No revocation hook; it also holds after a crash. + +| Chord | The app | +| --- | --- | +| stamps principals on links | what a grant is, share links, cookies, `authenticate` | +| keeps one object per principal key | defines `Revocation` and `revokedBy` | +| on `revoke(r)`: ends matching objects and links, passes `r` to children | decides when to revoke; stores who is blocked | + +**Check questions** + +1. Why must the label be passed on to the worker? +2. The owner archives S1. Which call does the app make, and whom does it hit? +3. The server restarts the day after guest g1 was removed. What keeps g1 out? + +*Model answers:* (1) Guest g1's `Notes {S1}` lives on the worker; ending objects only on the server would leave the +view alive there. (2) `revoke({ session: "S1" })`: every guest of S1 (each carries the label `{ session: "S1" }`); the +owner carries no such label. (3) Not Chord: after the restart nothing live exists; the guest's tab must authenticate +again, and the app's stored record of grant g1 is blocked, so `authenticate` refuses it. + +--- + +## Concept 12: `ctx` + +**The idea.** Every call gets a `ctx` as its last argument: + +- `signal`: fires when the caller cancelled or the object ended (there is no per-call deadline); +- `principal`: the caller (remote types only); +- `get`: references that are released when the call ends; +- `telemetry`: the current trace span; `ctx.trace(name, attributes, fn)` runs `fn` with a child `ctx` under a child + span (concept 29); +- in a job (concept 17) also `jobId` and `progress`. + +Pass `ctx` or `ctx.signal` to everything you wait on (`fetch(url, { signal: ctx.signal })`). Code outside objects gets +a `ctx` from `host.root(fn)`. There are no code versions and no request ids in it. Durable's `Context` is the same +module, so passing between them needs no conversion. + +**Check question:** a caller closes its tab while `exportText` runs. How does the method learn it? *Model answer:* +`ctx.signal` fires; everything it awaits with the signal is cancelled. + +--- + +## Concept 13: Documents and transactions + +**The problem.** Notes must be stored, changed safely while others read them, sent to viewers as small changes, and +survive restarts. + +**The idea.** An object's data lives in **documents**: named JSON values the object owns, changed only in +**transactions**. + +```ts +const NoteList = defineState("notes.list", { + schema: NoteListSchema, // { items: Note[] }: a document's root is an object + version: 1, + initial: () => ({ items: [] }), + persist: true, // stored; default: memory only +}); + +host.serve(NoteStore, ({ session }, self) => { + const list = self.state(NoteList); // a handle to THE list document of this object + return { + notes: list, // a singleton handle is a readable: expose it as a member + add: ({ text, by }, ctx) => self.tx(async (tx) => { + const d = await tx.draft(list, undefined, { tags: [SessionTag(session)] }); // the current list, as a draft + const id = uuidv7(); + d.items.push({ id, text, by, at: Date.now() }); + return id; + }, ctx), + remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), + }; +}); +``` + +- **Singleton or keyed:** a singleton (no `keyed`) is loaded at activation and kept in memory; `list.value` is + synchronous. A `keyed: true` document exists many times per object, addressed by a short key, loaded on demand + (`await handle.get(key)`; `handle.keys()` pages keys from storage). There is no live key list: a collection you show + live is a singleton; large collections are methods returning pages. +- **Shape:** a document's root is always a JSON object (as in Durable); a list lives in a field (`items`). Documents + hold JSON only. +- **Only `tx.draft`** (as Durable's `tx.doc`): `await tx.draft(handle, key?, { tags? })` returns a draft of the current + document, or a draft over `initial()` if it is absent. Reading is drafting: the draft is an overlay, nothing is + copied. A draft you don't change commits nothing, so reading never stores an absent document. `tx.retire(handle, + key?)` removes one; a retired singleton reads as `initial()` again. There is no `tx.get`. Shorthands: + `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). +- **Values are immutable by convention:** `list.value`, values read in a transaction and published values are shared; + nobody mutates them, and nothing is copied or frozen outside development mode. +- **Handles belong to their object:** they may be used only in transactions of their own object (or of the facet that + serves it, concept 18). +- **Commit** (Durable's sequence): wait for the object's line (one transaction at a time per object), run the + callback, write all persisted documents in one storage batch (the change as a delta, or a full base now and then), + adopt the new values by pointer swap, notify local subscribers **once** with the commit's operations, send **one** + batch per link. An error or abort before the write changes nothing. Nothing is revalidated or copied. +- **Durable's rule:** inside the callback only `await tx.*`. No other objects, network, models or processes, no second + `self.tx` (otherwise the object's line hangs). Fetch first; decide and write inside. +- **Versions:** a changed shape or meaning raises `version` with a pure `migrate(old, from)`; documents migrate on + access. Old code meeting a newer document throws; without `migrate` access is refused and the data kept. A failing + load or `migrate` fails the whole owning object (`broken`, concept 7). +- **Schema fingerprints instead of value checks:** each definition computes a fingerprint (a hash) of its schema once; + every stored document records the fingerprint it was written with. On access Chord compares the two: same version + but a different fingerprint fails the object `broken` ("schema changed without a version bump"). That costs nothing + per change; values are not walked against the schema in production. So every schema change needs a version bump + (an additive one gets a trivial `migrate`). A changed meaning without a schema change still cannot be detected. +- **Tags** (`defineTag(name)`, e.g. `SessionTag`) are set on create; deleting a session retires every document with + its tag. +- **`persist` only in `local` objects and `lifetime: "host"` objects** (identities without a principal). A `remote` + object's identity contains the principal key (how the client connected, the grant revision), so its stored documents + would scatter across access paths and be orphaned after a reconnect. The type checker enforces it (`self.state` in a + remote factory accepts only non-persisted definitions), and at runtime the declaration fails with "persist needs a + stable identity: put the documents into a local shared store". +- **Lifetime:** an in-memory document ends with its object (last reference released, linger expired); it survives + reloads and native restarts of a live object. What must outlive the object is persisted, or lives in a + `lifetime: "host"` object. +- **What survives a restart** is exactly what is persisted. Everything else (in-memory documents, native values, + derives, routines, references) is rebuilt on activation. +- **The store** is a port of Durable's store design: lines, commits, migrations, tags and eviction (a keyed persisted + document unused for 10 minutes leaves memory). Behind it is a `DocumentStorage`, passed to `createHost({ storage })` + (in memory by default). Chord ships two: in-memory and SQLite (on Chord's SQLite drivers, which moved there from + Durable; a Node driver, a Durable Object driver). Storages check only metadata and write the delta; they never + rebuild or validate content on a commit. The server has one shared storage; each worker one for its session. +- **A storage error** ends the store and closes the host; `onError` gets `{ kind: "storage_failed" }` and the app's + start code decides (exit with its own code for its supervisor, or reopen). Chord never ends the process. + +| Kind of state | For | API | +| --- | --- | --- | +| document | data the object owns | `self.state`, `self.tx` | +| derive | computed values | `self.derive` | +| source readable | external data (Durable views, files, logs) | value plus ordered changes | +| paged method | large collections | a method returning pages | +| plugin database | full-text search, large blobs | a native value (concept 14) | + +**Check questions** + +1. Why must a network call that fetches a link title happen before `self.tx`, not inside it? +2. A viewer watches the list while `add` runs. How many updates does it get? +3. Why may the per-principal `Notes` not persist documents, while `NoteStore` may? + +*Model answers:* (1) Inside a transaction only `tx` operations are allowed; anything else can hang the object's line. +(2) One, after the commit. (3) `Notes` is `remote`: its identity includes a principal key that changes (connection, +grant revision), so stored data would be orphaned; `NoteStore` is a local shared store, its identity is (type, input). +The type checker refuses a persisted document in a remote factory. + +--- + +## Concept 14: Native values (`self.open`) + +**The problem.** The inbox needs a file watcher: a **native value** (an OS-level thing: a file handle, a socket, a +process, a database connection). It must be opened, closed exactly once, never leak, and cannot be JSON or copied. + +**The idea.** Native values live only in objects. An object declares one with `self.open(async ({ signal }) => value, +(value, reason) => close)`; it gets back a handle whose `.value` is the native value (reading it before activation +throws). Chord opens natives one after another in declaration order during activation (the object goes live after all +opens; a later opener may use an earlier handle's `.value`) and closes them in reverse order when the object ends. +**The value never changes during the object's life.** + +```ts +// worker.ts: one inbox per worker +host.serve(NoteInbox, (_input, self) => { + const info = self.get(SessionInfo, {}); // start data (concept 5) + const watcher = self.open(async ({ signal }) => { + const { id, dataDir } = info.init.value; // read in the opener, after activation + return watchDir(join(dataDir, id, "inbox"), { signal }); + }, (w) => w.close()); + self.routines.loop("import", async (ctx) => { // concept 15 + const store = await ctx.get(NoteStore, { session: info.init.value.id }); + for await (const file of watcher.value.files(ctx.signal)) + await store.add({ text: await read(file), by: "inbox" }, ctx); + }); + return {}; +}, { lifetime: "host" }); +``` + +- **Why `lifetime: "host"` and no input:** the inbox must run as long as the worker, whether anyone looks or not, and + the facet cannot `self.get` a type it serves (concept 18). So a worker-wide owner of a native value is a + `lifetime: "host"` object without input that reads its session from the core's start data (`SessionInfo`). +- An **object with `self.open` is a native object;** one without is a **pure object**. The difference decides what a + reload does (concept 23). +- Behaviour around a native value goes into routines; listeners on native values outside routines are forbidden. +- **An open that fails** makes the object `failed`: `transient` with backoff, or `broken` if the opener says so + (`throw broken(error)`). A partly failed activation closes the natives already open; the next attempt opens all of + them anew. +- **`close` gets a reason:** `{ kind: "reload" }` (the plugin is reloaded and the object restarts, concept 23), + `{ kind: "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). + Chord aborts only the object's own calls and jobs; other work using the native value is the owner's business, so + `close` may drain instead of tearing down (code mode seals its sandbox and ends it when running scripts finish). + Each `close` is bounded (30 s by default, then reported as a possible leak); a throwing `close` does not stop the + others. +- **Natives that follow a relocation** (the session moves to another directory): the native is a stable manager + opened once; a routine on the session location switches its internal resource (the watched directory) and bumps a + generation, so late events from the old location are dropped. +- **No native values across plugins,** except through registry entries (concept 19). The facet may not open any. +- What an old native value does after its owner restarts is its owner's `close` (a model runtime keeps working until + collected; a closed SSH connection fails its in-flight `exec`). +- **A native value for one call** (a temporary SSH session for a directory listing) is opened and closed inside the + method with `try/finally`; there is no `ctx.open`. + +**Check questions** + +1. Why is the watcher not opened at the top of `worker.ts`? +2. What does "the value never changes" spare the author? + +*Model answers:* (1) Native values live only in objects, which open and close them with their life; a module-level one +would leak across reloads and have no owner. (2) No stale handles, no re-reading at use, no replacement logic: if it +must change, the object restarts. + +--- + +## Concept 15: Routines + +**The problem.** Delete old notes every hour; import inbox files as they arrive. Bare timers and callbacks would keep +running old code after a reload and touch state from nowhere. + +**The idea.** Standing behaviour is declared as a **routine**: + +```ts +self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads +self.routines.on(settings, (value, ctx) => applyLimit(value, ctx)); // react to a readable or event +self.routines.loop("import", async (ctx) => { /* runs until the object ends */ }); +self.routines.custom("ticker", ({ signal, run }) => { + const t = setInterval(() => run(tick), 1000); + return () => clearInterval(t); // cleanup undoes the setup +}); +``` + +- Started when the object goes live, stopped when it ends, restarted with the new code on every reload of its plugin. + Routines are named, so Chord recognizes them across reloads. Each run gets its own `ctx`; errors are caught and + reported. +- **Serialized per routine:** the new instance starts only after the old handler in flight has settled (with a + deadline, 10 s by default; a stuck handler is reported). Without this, an old handler midway through a restart and + the new routine could both act on the same item. +- A routine **reconciles** from the current value ("make the world match this"), writes no "in progress" flags, and + starts a job for long work. A handler that only cares about the latest value compares with the current value itself. +- No bare timers or callbacks that touch state outside routines, jobs or calls. + +**Check question:** a reload happens while `cleanup` is halfway through. What happens? *Model answer:* the new routine +waits until the old handler finishes (or the deadline passes), then starts with the new code; the due time is kept. + +--- + +## Concept 16: Rules across owners + +**The problem.** Every thing has exactly one **owner**: the smallest object it belongs to, which holds its data, +enforces its rules in its methods and offers the operations others need. But some rules span two owners: "notes of an +archived session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). + +**The idea.** + +- **No plugin reads or writes another plugin's documents** (Chord or Durable). Data crosses plugins only through state + and methods. Within a plugin, the facet's documents are readable and writable by the objects that facet serves. +- **The owner of the dependent data enforces the rule** by reconciling from the other owner's state with an idempotent + routine. No cross-owner transactions, no "notify the other owner" calls that can be lost: + +```ts +// NoteStore: "an archived session's notes are read-only" +const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document +const catalog = self.get(Catalog, {}); // the server's session list (another owner) +self.routines.on(catalog.sessions, async (sessions, ctx) => { + const archived = sessions[session]?.archived ?? false; + if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); // idempotent +}); +// add/remove refuse while flags.value.archived +``` + +After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. + +- **Reading without starting a worker** is plain publishing between two objects of one plugin, no Chord mechanism: + the plugin's worker facet keeps a summary (say, the note count) and a routine reports it to the plugin's server + object by a method call; clients read the server object, which never starts a worker. The server object keeps the + latest value per session (a document if it must survive a server restart) and ignores reports from an older worker + incarnation. The session list, status, unread marks and artifact lists work this way. + +- **Data core enforces belongs to core:** it is handed to core through core's API when the data is created (Durable + `createConversation` with `init`), never written into core's documents directly. + +**Check questions** + +1. Why doesn't the catalog call `notes.archive(session)` when a session is archived? +2. The worker crashes right after the session was archived. Is the rule still enforced afterwards? + +*Model answers:* (1) The catalog would have to know every plugin, and a call can be lost (crash, link drop); the +dependent owner reconciles instead. (2) Yes: on activation the routine sees the current catalog state and sets +`archived`. + +--- + +## Concept 17: Jobs + +**The problem.** Importing a 50 MB file takes a minute. The call that started it may end (tab closed); the work must go +on, be cancellable, not run twice on a retry, and its result must be recorded. + +**The idea.** A **job** is owned work started by a method, in three steps (Durable's "sandwich"): + +```ts +const ImportJob = defineJob("notes.import", { + version: 1, args: ImportArgs, progress: ImportProgress, result: ImportResult, concurrency: 1, +}); + +const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { // 2. effect: may do anything; keeps its code on reload + let n = 0; + for await (const line of readLines(path, ctx.signal)) { + await add({ text: line, by: "import" }, ctx); + ctx.progress({ n: ++n }); + } + return { n }; + }, + finish: async (outcome, tx) => { /* 3. outcome, in the current code, Durable's rule */ }, +}); +return { + imports: imports.running, // live: [{ id, version, args, progress, phase }] + startImport: ({ path, requestId }, ctx) => imports.start({ path }, ctx, { key: requestId }), // 1. intent + abortImport: ({ id }) => imports.abort(id), +}; +``` + +- **1. Intent** is a transaction: the busy check (`concurrency`) and deduplication (`key`: the same key returns the + existing run) are atomic; `run` starts only after the commit. Run ids are uuidv7. +- **2. Effect:** `run` may do anything; external writes carry a dedupe key derived from `ctx.jobId`. On reload it + keeps its old code. +- **3. Outcome:** `{ ok: true, result } | { ok: false, error: "threw" | "aborted" | "lost" }`. Chord records it, then + calls the **current** code's `finish(outcome, tx)` and acknowledges in the same commit. A throwing `finish` is + retried. +- **Abort is a request:** it fires `ctx.signal`; the run decides what it returns. +- **Waiting:** `start` returns a run with `outcome(ctx)`; the caller's abort ends only its wait, never the run. +- **`phase`** in `running`: `running`, `finishing` (outcome recorded, `finish` not yet committed), `blocked` (no + adapter for an old version, concept 26). +- **Jobs keep their owner alive:** a running, finishing or blocked job counts as a reference to its object; the object + ends only once all its jobs are settled. Starting an import and then closing the tab does not abort it. +- **Persistence:** in memory by default. `persist: true` keeps the record: a run live at a crash ends `lost` at the + next activation. Nothing is rerun automatically. +- **Not jobs:** request/response work (a method), input into running work (concept 22), work that must resume after a + crash (a Durable task). + +> **Open: jobs as objects.** A login or a long relocation is a job whose state consumers want to address and watch +> (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; (b) a job is an +> object of its own, addressed by its id, living until done. + +**Check questions** + +1. The tab sends `startImport` twice with the same `requestId` (retry after a link blip). How many imports run? +2. A reload of the notes plugin happens during an import. Which code runs the rest of `run`, which runs `finish`? + +*Model answers:* (1) One: the key dedupes atomically. (2) `run` keeps its old code; `finish` runs in the new code. +(`NoteStore` is a pure object; in a native object the run would end `aborted`, concept 23.) + +--- + +## Concept 18: The facet + +**The problem.** A plugin needs per-host setup: serve its types, contribute to extension points, maybe keep some +plugin-wide data. + +**The idea.** The **facet** is the plugin's setup for one host, and itself a **pure object** that lives as long as the +host: + +```ts +// notes/server.ts +export default defineFacet((host, self) => { + const prefs = self.state(NotesPrefs); // plugin-wide documents + self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); // concept 19 + host.serve(SomeServerType, (input, principal, self) => …); +}); +``` + +- It may hold documents, references, routines and contributions; **no native values**. +- **Its identity is the plugin's `package.json` name** (setting): its persisted documents are found by that name, so + they survive restarts and reloads, and never depend on a directory or the load order. +- Served objects go live after the facet and may read and write the facet's documents; one transaction may span an + object's own documents and its facet's (lock order: own line, then the facet's). The facet never reaches into a + served object's documents, and never `self.get`s a type it serves itself. +- `host.serve` and contributions are valid only while the setup runs. +- **Between plugins** only two things pass: object types and registries. + +**Check question:** why does the facet not open the inbox watcher? *Model answer:* the facet is a pure object; native +values live only in objects with a lifetime of their own, which open and close them. + +--- + +## Concept 19: Registries + +**The problem.** The notes plugin adds a download route; a logs plugin wants to time every request; several plugins +add model providers. Someone must combine these contributions, decide conflicts, and replace a plugin's part as a +whole on reload. + +**The idea.** A **registry** is an extension point owned by someone (a Chord component, the app, a plugin). Each +contributor (a facet or an object) contributes **at most one value** per registry; the table is an immutable, ordered +list of `(contributor, value)`; the registry's **`compose`** turns it into what consumers use: + +```ts +export const Http = defineRegistry("chord.http", { + version: 2, + accepts: { 1: (old) => upgrade(old) }, + compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value, reports conflicts +}); + +self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor +const router = host.registry(Http).value; // a consumer: live +``` + +- **`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or + ends, when a contribution changes (contributions may themselves be readables, known only once a native value is + open). Because contributions can change at any time, no check before a reload could be complete, so conflicts are not + a reason to roll back. +- **It always delivers a value,** with its fallback built in (e.g. the first by plugin name wins), and reports + conflicts through `report` (they become notices to the owner and telemetry events). Readers get a plain value and + never handle conflicts. Only if `compose` throws is the registry in an error state; readers then get an error when + they use it. Work that started earlier keeps the value it read. +- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an + object's from `live` until it ends, fails or restarts. +- **The owner decides** conflicts, order and which contributions apply where. Helpers: `keyed(...)`, `ordered(...)`. +- **Optional dependencies** are registries owned by the dependent (concept 8): ports contributes its forwards to + `radius.forwards`; without ports the table is simply empty. +- **Contributor order** is sorted by plugin name (the `package.json` name): the same on every host and across + reloads, and deliberately meaningless. Meaningful order comes only from the owner's composition (for HTTP: phases, + concept 20). +- **Versions:** a contribution carries the registry version its plugin was built against; older ones are adapted with + `accepts`, newer ones are refused at load. +- **Registries may carry native values.** An entry lives exactly as long as its contributor. Consumers look up at each + use (or follow the composed value) and keep nothing from an entry beyond a use. This is the only way native values + reach other plugins. Example: the models plugin's object contributes `() => runtime`; the agent harness looks it up + at every request; after a reload the next request gets the new runtime, while a running stream finishes on the old + one. +- Registries add behaviour (tools, routes, renderers, commands); they never replace an owner's checks. Entries have no + lifecycle: anything needing cleanup is a native value of the contributing object. + +**No callbacks between objects.** Members carry data only, on both kinds of types. A callback would carry code and +whatever it closes over from one object into another. Example: `NoteInbox` passes `onFile` to another object, which +keeps it. After a reload of the inbox's plugin, that object still calls the **old** `onFile`; after `NoteInbox` ends, +it calls into an object whose native values are already closed. The replacements: behaviour from outside goes through a +registry (the entry lives exactly as long as its contributor and is looked up at each use), multi-step work through a +job, input from a person through a flow (concept 22). + +**Check questions** + +1. Notes and files both contribute `GET /download`. When is that noticed and what happens? +2. A route handler stores `const r = entry()` in a module variable and uses it for days. What's wrong? +3. Object A passes a callback to object B, which keeps it. What does B call after A's plugin is reloaded, and after A + ends? What do you use instead? + +*Model answers:* (1) When `compose` runs after the second contribution enters the table (at runtime, not in a reload +check): the router still works, the first by plugin name wins, and the conflict is reported to the owner and as a +telemetry event. Nothing is rolled back. (2) It keeps a value from an entry beyond a use; after the contributor +restarts it holds a stale or closed native value. Look up at each use. (3) After the reload, A's old code; after A ends, code of an object +that no longer exists, with its native values closed. Instead: a registry entry, a job, or a flow. + +--- + +## Concept 20: HTTP routes and wraps + +**The problem.** Every plugin that serves a download would repeat ranges, HEAD requests, download headers, streaming +and abort on disconnect, and get some of it wrong. + +**The idea.** HTTP is an optional Chord component (`@earendil-works/chord/http`) with a registry. It is built +after the kernel and a first real proof (harness, one tool with an approval pause, a code mode reload); until then the +playground serves its pages with plain app code. The design below is the target. A route gets a +plain request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, +parsed and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's principal (from the +request resolvers, which decide who sent a request: cookie, loopback, relay). So `ctx.get(Notes, …)` gives exactly the +view that person may see. The route returns a **description**; the component does the HTTP work once: + +```ts +| { kind: "json"; value } | { kind: "page"; title; body } +| { kind: "bytes"; name; mime; size; read(offset, length) } +| { kind: "redirect"; to } | { kind: "error"; status } +``` + +```ts +// notes/server.ts: a route is (req, ctx) => Promise; ctx.principal is the request's principal +self.contribute(Http, { + routes: { + "GET /s/:session/notes.txt": async (req, ctx) => { + const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view + const text = encode(render(notes.notes.value)); + return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, + read: (offset, length) => text.subarray(offset, offset + length) }; + }, + }, +}); + +// logs/server.ts: a wrap (middleware) around every route +self.contribute(Http, { + wraps: [{ + name: "access-log", phase: "observe", scope: "all", + wrap: async (req, next, ctx) => { + const t = Date.now(); + const res = await next(req); + log(req, res, Date.now() - t); + return res; + }, + }], +}); +``` + +- **Routes** are composed with `keyed`: two contributors with one pattern is a reported conflict; the first by plugin + name wins. +- **Wraps** run in **phases the owner defines** (`observe`, `headers`, `transform`; outermost first), within a phase in + contributor order. That order is deterministic but meaningless, so wraps in one phase must not depend on each other. + No numeric priorities (Fastify's lifecycle phases are the model). +- **Scope:** a wrap applies by default only to its own contributor's routes (`scope: "own"`); `scope: "all"` is + explicit; `route` narrows it to one route, and a `route` that does not exist is a reported conflict. +- Wraps may add a security boundary on top of an owner's checks, never instead of them. +- A request uses the router as it was when the request started. +- A **channel component** provides WebSocket byte channels (`send`, `onData`, `close`) for terminals, with their own + flow control, separate from object links. + +**Check questions** + +1. Who handles a `Range: bytes=1000-` request for `notes.txt`? +2. The logs plugin adds a wrap without `scope`. Does it see requests to `notes.txt`? + +*Model answers:* (1) The HTTP component: it calls `read(offset, length)` and sends 206; the route only describes the +bytes. (2) No: the default scope is `own`, only the logs plugin's own routes; it needs `scope: "all"`. + +--- + +## Concept 21: Who is here: presence + +**The problem.** Show "2 people viewing"; keep the worker alive while someone looks; know when a login flow's owner +left. + +**The idea.** `host.references(Type)` is a readable of who holds references to objects of a type on this host (input, +principal, client). Each tab is its own client (an origin id the client cannot set). + +```ts +const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => refs.length); +``` + +- **A link drop is a departure:** accurate and simple. Smoothing (a phone switching networks flickers the count) is + the UI's job, e.g. a debounced viewer count. +- **Counts per person** are derived by the app from the principals and origins in the list; two tabs of one person + are two clients. +- **No lingering on the consumer side:** a closed tab closes its link, so it cannot keep anything. What keeps a + just-closed object warm is the provider's `linger` (concept 7), and that also keeps the worker alive (concept 4). +- "Leaving" is never derived from a component unmounting (concept 23), only from navigation or presence. + +**Check question:** the owner closes the notes tab and reopens it ten seconds later. Is the viewer count briefly +lower, and does `Notes {S1}` have to be rebuilt? *Model answer:* yes, the count drops while the link is gone (the UI +may smooth it); no rebuild if the type has a provider-side `linger` longer than ten seconds: the object stayed warm. + +--- + +## Concept 22: Input into running work: flows + +**The problem.** A login asks for a pasted code, then an account choice; a tool asks for approval. The running work +waits for a person who may be on any tab, may reconnect, may answer twice. + +**The idea: the owner keeps the flow as state.** + +```ts +start({ provider }, ctx) → { id } // uuidv7 +LoginFlow {id}: state { status: "running" | "done" | "failed" | "cancelled", step?: { id, kind, … }, notices } +answer({ id, step, value }, ctx) → ok | "step no longer active" +cancel({ id }, ctx) +``` + +- The UI watches the state; a late or second answer to a step that is gone is refused. +- A promise-based callback (pi-ai's `prompt`) stays **inside** the owner (its job keeps the resolver); nothing crosses + an object boundary. +- **Imperative consumers** (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not + a second protocol: + +```ts +for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) + await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); +``` + +- Another pattern is the shared **Questions service** (`questions.md`): typed questions, policies over a live + audience (`first`, `quorum`, `all`), reattach by `key`, deadlines. Its store must live in an object. + +> **Open: where a flow lives.** Options: (a) a keyed document of its owner (`Login`); (b) an object `LoginFlow {id}` +> living until done (tied to "jobs as objects", concept 17). + +**Check question:** why not a server method `step()` that blocks until the next step? *Model answer:* a link drop fails +a long call and the consumer can't tell whether it missed a step; the helper reads state that re-holds by itself and +remembers the last step id. + +--- + +## Concept 23: Reload + +**The problem.** New code must take over without losing data, open watchers, running imports or connected tabs, and a +broken change must not leave a host half-updated. + +**The idea, per host:** + +``` +1. load the plugin's new bundles; an import or syntax error rejects +2. staging facet setups and every live object's new factory run with a recording `self` + (declarations only, nothing activated); a throw rejects +3. commit the host switches to the new code atomically +4. after per object, below +``` + +- **Rollback is all-or-nothing per host:** any code error in load or staging (an import error, a throwing factory) + leaves the whole host on its previous code. A host is always completely at one revision, so its status is one value. + Runtime states (a registry conflict, a failed open, a migration) are not reasons to roll back. +- **Pure objects** are swapped in place: new methods and derives; documents, references and jobs stay; routines + restart (serialized); new references become live; then the gate opens. Running calls finish on the old code and keep + their old handles valid until they settle (their transactions keep working); only new calls get the new handles. +- **Native objects of the reloaded plugin restart:** their running work ends `aborted`, native values close (reason + `reload`), the new factory runs, native values reopen. Documents stay (they belong to the identity). In practice this + costs little: the Durable harness is base code, terminal processes live in a separate host process, and model + runtimes, sandboxes and the search index rebuild in milliseconds; only reloading the plugin that owns SSH + connections (running remote commands are killed) or the relay socket (relayed connections drop) hurts. +- **Why natives are not kept across a reload by name:** a native value has no line, no transactions and no schema, so + old and new code would use it interleaved without any check. +- **After the commit,** errors stay with one object: a failed open makes it `failed` (backoff); a migration that throws + refuses that access; a method bug fails that call. +- **Removing a type** ends its objects; in-memory documents go; persisted ones stay until retired. +- **Consumers on other hosts** see a restarting object like any object that briefly goes away. +- No factory trial, no migrate dry run, no consumer check in staging: an incompatible consumer fails at bind with + `contract_mismatch` and a notice naming plugin and type. +- **Start barrier** (host start, first load): load the facets (the core facet with its start data among them), wait + until every facet and every `lifetime: "host"` object is live or failed, then resume dependent work (the Durable + harness, concept 28). An object that must contribute at start (the model runtime) is therefore `lifetime: "host"`. + +**In the notes plugin:** you change the cleanup interval and the reader's filter, and reload. `Notes` and `NoteStore` +are pure: swapped in place, the notes stay, open tabs keep their references, a running import keeps its old `run`. +`NoteInbox` has a native value: it restarts, the watcher closes and reopens. + +**Loading and the dev loop.** Plugins come from built-in, global, package and project roots, found by directory +listing; the server selects; dependencies resolve by contract name at runtime; names are prefixed (`.`). +The loader is an adapter: one base bundle per surface plus one script per facet (loader D, which reads each plugin's +`package.json` name), or static imports (tests, Durable Objects), where the app passes the same name to `load`. A +source watcher only marks plugins dirty; a reload control (web shell, CLI, agent tool) reloads them or restarts all +hosts for base code. + +**UI reload vs. leaving:** a reload of a UI facet unmounts and remounts components. "Leaving" (abort a login, close a +TUI overlay) is therefore never derived from an unmount, only from explicit navigation or presence (a dropped +reference). + +> **Open: project plugins.** Per-directory plugins with trust and snapshots (`plugin-loading.md`) need their own pass +> over this design. + +**Check questions** + +1. A typo in the new `NoteStore` factory throws in staging. What runs afterwards? +2. Which notes objects restart on a reload, and why only those? +3. Why is rollback per host enough, instead of per plugin? + +*Model answers:* (1) The whole host keeps its previous code; nothing changed. (2) Objects with a native value +(`NoteInbox`): a native value can't be swapped safely in place; pure objects are swapped. (3) A host is then always +completely at one revision, so its status is one value per host. + +--- + +## Concept 24: Rollout across hosts + +**The problem.** A reload touches the server, every worker and every client. Calls between hosts keep flowing while +they update one by one. + +**The idea.** A **revision** is (server incarnation, sequence) and names the complete desired code set (every facet +with its digest). The rollout goes server first, then workers, then clients. Each host reports `installed`, +`installed_failed` (named failed objects) or `rejected` (with the reason; it stays on its previous code). A server that +rejects stops the rollout. A host that starts or reconnects installs the current revision before anything is routed +to it. + +A call that reaches a provider on another revision gets the provider's revision and status: + +| Provider | The call | +| --- | --- | +| ahead, or behind and installing | `updating`: values readable, calls park (60 s deadline, concept 8) | +| rejected | fails at once (retried on the next revision) | +| same revision | `contract_mismatch`: a real mismatch, final | + +- **A host checks its own install status first:** if its own install was rejected, a call fails at once with the + reason instead of waiting on a provider. +- **Installs are serialized per host.** +- **A revision carries a digest of the code set** (the hash over the facet bundles' content hashes, which the loader + computes anyway). Hosts compare it on reconnect, so a server restart with unchanged code triggers no reinstall. + +**Check question:** the server installed revision 8; worker S1 is still installing. A tab calls `Notes {S1}` with the +new contract. What happens? *Model answer:* the call parks until S1 reports revision 8, then runs. + +--- + +## Concept 25: The wire + +**The idea.** Exposed documents, derives and readables are state members on the wire: `snap` when a reference is held, +`ops` per commit (the draft's recorded operations, no diff), `ops` from diffs for derives, throttled `ops` for job +progress, `reset` after a migration, `error` for a failed member. + +- **Commits are atomic for consumers:** one `state` frame per hold carries all member updates of a commit, applied + before listeners run; a commit that touches several holds on one link travels as **one** frame. No consumer ever sees + half a commit. +- **Compression:** large JSON envelopes are deflated on WebSocket links; byte segments stay raw; IPC never compresses. +- **Frame size limit** per link adapter (e.g. 16 MiB on client WebSockets, none on IPC): an oversized frame fails that + call or member with `too_large`; the link stays up. +- **One encoding per frame:** a frame is encoded once per link and its size taken from the encoded bytes. **Relays** + read only the frame header (hold id, kind, route) and forward the body without decoding it. **Receivers** check a + frame once by its header; only untrusted callers' arguments are validated (concept 6). **Consumers** apply changes + with structural sharing; nothing is validated or cloned. +- **Contract descriptions travel once per link:** a new reference sends only its type's fingerprint; the provider asks + for the full description once per (link, fingerprint) and caches it for that link. +- **Flow control** per link: a byte window on WebSockets, a window of frame counts on IPC, applying to every frame. + Calls are throttled by credits the provider grants; beyond them they wait locally; nothing is refused for being "too + many". A relay that falls too far behind releases the hold and holds again; the provider sends a fresh snapshot. +- **Adapted bindings get diffs:** a consumer bound through an `accepts` adapter (concept 26) is served like a derive: + Chord keeps the last adapted value per binding, applies the adapter to the new value and sends the difference. +- The link protocol has a version; hosts of different versions refuse each other (a web page reloads). + +**Check question:** a guest adds a note; the owner's tab holds both `Notes {S1}` and a counter on the same worker. How +many frames reach the owner's tab for that one commit? *Model answer:* one `state` frame with both holds' batches. + +--- + +## Concept 26: Versions + +**The problem.** Parts change independently: a plugin built last month talks to today's provider; a document stored a +year ago meets new code. + +**The idea.** Two kinds of versions: + +- **Interface versions** (between parties that change independently) have adapters: object types (`version`; + same version binds if the provider covers the caller; otherwise `accepts` keeps older consumers working, or they get + `contract_mismatch`; the **fingerprint**, a hash of the type's description, covers input, variants and members, not + placement), registries (`accepts` on contributions), jobs (`accepts` on outcomes; without an adapter the run is + `blocked`, not refused). +- **`accepts` maps values per member,** pure functions: + + ```ts + accepts: { 1: { state: { notes: (v2) => v1 }, methods: { add: (argsV1) => argsV2 } } } + ``` + + Chord wraps a state mapping like a derive and sends diffs per binding (concept 25); members an old consumer doesn't + know don't appear. +- **Internal versions** (old and new code of the same owner) migrate: documents with a pure `migrate`, guarded by the + schema fingerprint (concept 13). + +| Surface | On change | +| --- | --- | +| object type | `accepts` adapter; coverage check | +| registry | `accepts` adapter on contributions | +| document | pure `migrate`, on access; a schema change without a bump fails the object `broken` | +| job | `accepts`; `blocked` without an adapter | +| native value | none: the object restarts | +| Durable tasks and documents | Durable's `migrate` | +| link protocol | refused; hosts update together | + +**Check question:** why do documents get `migrate` but object types get adapters? *Model answer:* a document is shared +only between its owner's old and new code, so it is converted once; an object type is used by consumers that update on +their own schedule, so the provider keeps serving old shapes. + +--- + +## Concept 27: Bytes and big transfers + +**The problem.** A user uploads a 50 MB file to import. Base64 inside JSON grows it by a third; one giant message +blocks the link and fails as a whole on a drop. + +**The idea.** + +- **Bytes are a schema type** (`bytes()`, a `Uint8Array`), carried natively: references on memory links, structured + clone on IPC, binary frames on WebSockets. Never base64. +- **Big transfers are chunked by the owner,** not by Chord: + +```ts +beginUpload({ name, size }, ctx) → { upload } +append({ upload, offset, bytes }, ctx) // about 1 MiB; awaited = backpressure +finishUpload({ upload }, ctx) → { id } +read({ id, offset, length }, ctx) → bytes +``` + +- The awaited `append` is the backpressure; after a link drop the writer asks for the stored offset and resumes; an + unfinished upload expires (the owner's routine). A small helper wraps the loop. +- Chord has no stream type: it would need its own rules for link drops, cancellation, relays and restarts, which the + owner's offset protocol already survives with plain calls. + +**Check question:** the link drops after 30 of 50 chunks. What happens? *Model answer:* the `append` in flight fails +like any call on a drop; after reconnecting the writer asks the owner for the stored offset and continues from chunk +31. + +--- + +## Concept 28: The Durable harness + +**The problem.** The agent runs on Durable (the transcript and task engine). It needs model access and execution +environments that come from plugins, and those plugins reload. + +**The idea.** The harness is an **object in the core facet** (base code, so it restarts only with the host): a +`Harness` object with `lifetime: "host"` (it runs for the worker's whole life, whether or not anyone holds it; its +activation is the start barrier) opens Durable with `self.open` and gets models and environments from registry +entries, looked up at each use: + +```ts +const harness = self.open(async () => Durable.open(storage, { + models: () => host.registry(HarnessModels).value, // current entry at each use + env: (target) => host.registry(HarnessEnvironments).value(target.cwd), + settings: () => currentSettings(), + registry, +}), (h) => h.close()); +``` + +- Durable's `models` and `settings` accept functions, so each generation gets the current value. +- Its activation is the **start barrier:** open Durable without scheduling, wait for the required contributions, check + cwd and model, then resume. +- After a models reload the next request uses the new runtime; a running stream finishes on the old one. +- Durable and Chord v2 share `Context`, `Draft` and `JsonValue` (same modules); only plain data crosses between them. +- Hooks such as tool approval open a Chord call scope with `host.root` inside Durable's hook. +- **Code mode** (a plugin) owns its sandbox as a native value. Its tool entry looks up the **current** sandbox when a + call starts, so a call that was waiting (e.g. in an approval hook) during a reload starts on the new sandbox; the old + one is sealed and drained in `close` (reason `reload` or `stop`). +- **Telemetry:** Durable gets the same `Context.telemetry`; submissions and tasks store a carrier (concept 29). +- App state lives in Chord documents; only transcript-bound data (entries that fork and rewind with a conversation) + lives in Durable. + +> **Open: a Durable Object profile.** Deploying Chord apps to Cloudflare Durable Objects: a DO can lose its memory +> while clients stay connected; routines need wake-ups (one alarm per DO); links between DOs are RPC stubs. Hot reload +> is not needed there, plugin composition is. For later. + +**Check questions** + +1. Why is `Harness` `lifetime: "host"` and not referenced like `Notes`? +2. Why does the harness look up `HarnessModels` at each use instead of reading it once when it opens? + +*Model answers:* (1) It must run for the worker's whole life, also while nobody holds it (Durable tasks continue), and +the start barrier waits for `lifetime: "host"` objects. (2) The models plugin may reload; its entry is replaced, and an +entry's value must not be kept beyond a use. The next generation gets the new runtime. + +--- + +## Concept 29: Telemetry + +**The problem.** A guest's `add` is slow. The call crossed the browser, the server and the worker. Where did the time +go? + +**The idea.** Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets one +adapter (`createHost({ telemetry })`, nothing by default) that decides sampling, buffering, export and filtering. + +- **The trace travels with the call.** `ctx.telemetry` is the current span. `call` and `hold` frames carry the + caller's **carrier** (an opaque `Record` the adapter writes and reads); the provider continues the + trace under it, and a relay adds its own span. An unknown or missing carrier just starts a new trace, never an + error. + + ``` + chord.call Notes.add [tab A] + └─ chord.relay [server] + └─ chord.method Notes.add [session:S1] principal=guest + · event chord.commit docs=1 ops=1 + ``` + +- **Own spans:** `ctx.trace(name, attributes, fn)` runs `fn` with a child `ctx` under a child span, so calls inside are + parented correctly. +- **Shared work** (one activation awaited by several callers, a worker start) is its own trace, **linked** to its first + trigger; each waiting span links to it. Routine runs, job runs, eviction and background closes are their own traces, + linked to their trigger where one exists. +- **Tiers:** spans for calls, methods, relays, holds, activations, opens, closes, loads, revocations, routine and job + runs; events for commits inside methods, retries and parking; metrics for high-frequency background work (state + frames, bytes, commit durations, in-flight calls, open links, park durations). +- **Nothing is measured just for telemetry:** only sizes the code already has (encoded frames, `byteLength`). +- **Durable** gets the same `Context.telemetry`; submissions and tasks store a carrier, so a task run links to its + submission even after a restart. Persisted Chord jobs store their start carrier. +- **Debugging:** a JSONL trace file adapter, a text renderer and a trace viewer in the playground. + +**Check question:** a routine in `NoteStore` runs every hour. Whose trace does its run belong to? *Model answer:* its +own (a root span), not a caller's; it is linked to its trigger where one exists. + +--- + +## Concept 30: The cost model + +**The problem.** A session's notes grow to 500 KB; ten tabs watch. If every added note made Chord walk, copy, validate +or re-encode the whole value per viewer, the app would crawl. + +**The idea.** As little compute and memory as possible while staying correct. Values are immutable revisions that +share structure (Delta, as in Durable), so each hop does only the work for what changed: + +| Hop | Work | +| --- | --- | +| commit | Delta's prepare (copies and checks only the placed values), adopt by pointer swap; storage writes the delta | +| reads in a transaction | the draft overlay, no copy; unchanged = no write | +| subscribers on the same host | the same value and ops by reference | +| derives | compute plus a diff; unchanged parts stay the same objects (`mapShared`, concept 9) | +| send | one encoding per frame and link | +| relay | header only, body forwarded unread | +| receive | header check; untrusted callers' arguments validated once at the provider | +| apply | structural sharing | +| new hold | the value encoded once (unavoidable) | +| memory | one authoritative value per document per host; replicas and derived values share structure | + +That is why several earlier concepts look the way they do: values are passed by reference and never copied (concept +6), documents are guarded by a schema fingerprint instead of value checks (concept 13), and derives must keep unchanged +parts identical (concept 9). + +**Cost guards:** in test mode Chord counts full-value walks, deep copies, encodes, schema validations and +materializations, and tests assert them per hop (one note appended to a 500 KB document with 10 viewers: no walks, no +copies, one encode per link, no validations). A benchmark suite records the numbers. + +**Check question:** a guest adds one note to a 500 KB list watched by ten tabs over one WebSocket link each. What does +Chord encode? *Model answer:* the commit's small ops, once per link; the 500 KB value is neither walked, copied nor +re-encoded. + +--- + +## Open points at a glance + +| Topic | Concept | +| --- | --- | +| jobs as objects | 17 | +| where a flow lives | 22 | +| project plugins | 23 | +| a Durable Object profile | 28 | + +App-level open points (p3, `design.md` section 26): session files, session delete, connection pools, config files with +a single writer, file formats as versioned interfaces, the terminal host process. + +--- + +## Glossary + +| Term | Meaning | +| --- | --- | +| host | one participant (a process, a tab): a logical identity plus an incarnation | +| link | a connection between two hosts; delivery only | +| relay | the server passing requests between clients and workers | +| host kind / address | typed description of addressable hosts / one host of a kind (`Session({ id })`) | +| placement | a type's `host` function: input → address | +| incarnation | a fresh id per host start; a restart is a new incarnation of the same identity | +| facet | a plugin's code for one host kind; a pure object living as long as the host | +| object type / contract | description: name, input, placement, members, version | +| `remote` / `local` | across links, per principal, untrusted arguments validated / own host, no principal, no checks; both pass values by reference | +| object | one instance per (type, input, principal key); `local`: per (type, input) | +| principal | who calls; stamped by links | +| principal key | maps a principal to an object identity | +| revocation | a small typed value the app revokes (`{ grant }`); `revokedBy(p)` says which ones hit a principal | +| variant | what a principal may see and do | +| shared store | a `local` object holding shared or heavy parts once, behind per-principal views | +| base code / core facet | the app's own code (`src/`), loaded under a fixed name such as `@app/core`; a change restarts the hosts | +| `SessionInfo` | the core's `local`, host-lifetime object holding the start data | +| reference | what `get` returns; keeps its object alive; live and hydrated when awaited | +| readable | a value plus its ordered changes | +| `ref.state` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | +| failure cause | `transient` (automatic backoff) or `broken` (retried only by a trigger) | +| `ctx` | per call: signal, principal, call-scoped `get`, telemetry span | +| document | named JSON value an object owns, written in transactions | +| derive | value computed from readables | +| `mapShared` | keeps one result object per unchanged source element, so derives stay cheap | +| schema fingerprint | hash of a document's schema, stored with every record; catches schema changes without a version bump | +| native value | an OS-level value opened with `self.open`; lives only in objects | +| pure / native object | an object without / with `self.open` | +| routine | standing behaviour: `on`, `every`, `loop`, `custom` | +| job | owned work: intent, run, finish | +| registry | extension point; one contribution per contributor; `compose` turns the table into a value | +| wrap | HTTP middleware, in owner-defined phases | +| flow | input into running work, kept as the owner's state, step by step | +| staging | reload step: facet setups and factories run as declarations only, nothing activated | +| revision | (server incarnation, sequence): the complete code set a host runs | +| carrier | opaque trace context carried in `call` and `hold` frames | diff --git a/apps/p2/docs/chord-walkthrough.md b/apps/p2/docs/chord-walkthrough.md new file mode 100644 index 000000000..9067373ce --- /dev/null +++ b/apps/p2/docs/chord-walkthrough.md @@ -0,0 +1,918 @@ +# Chord and p2: a guided walkthrough + +A step-by-step introduction to how p2 plugins work on top of Chord, for someone who knows nothing about either. It +describes the design (`target-design.md`), not the state of the implementation. + +--- + +## For the agent guiding the user + +You walk a user through this document. Follow these rules: + +1. **One concept at a time.** Explain it in your own words, in the user's language, with the example from the + document. Don't paste the whole section; keep it short and concrete. +2. **Then ask the check questions** of that concept (2–3) and **wait** for the answers. +3. **Judge the answers** against the model answers (they are for you, not for the user). If an answer is wrong or + half right, say precisely what is missing, explain that part again differently, and ask again. Don't move on while + a concept is not understood. +4. **Counter-questions come first.** If the user asks something, answer it completely before anything else. If the + answer lies in a later concept, say so, give a short answer, and offer to continue there. +5. **Move on only when the user says so.** +6. **No terms before they are introduced.** If you need one, introduce it first. +7. **Teach the design, not the implementation status.** Don't say what exists today and what is planned. When a + question goes beyond this document, answer from these sources, in this order: `target-design.md` (the design; + it wins on conflicts), `changes.md` (the reasoning behind each change), `packages/chord/docs/tutorial/` (Chord's + tutorial: plugin files, the app's API, how an app loads plugins), `packages/chord/docs/design.md` (Chord's API and + links). The code lives in `packages/chord` and + `apps/p2`; it may lag behind the design. +8. Keep a short list of what the user found hard, and come back to it at the end. + +Start by telling the user the note below, then begin with concept 1. + +## For the user + +**Keep asking until you really understand a thing.** There are no silly questions. The concepts build on each other: +a gap early on makes everything after it harder. If an example doesn't click, ask for another one. If a word is +unclear, ask what it means. + +--- + +## The setting and the running example + +**p2** is a coding-agent app built from plugins. It runs as several processes. Chord calls each process a **host**, and +every backend host has a name: + +- the **server** (name `server`): exactly one; serves the web UI, holds data shared by all sessions, starts the + workers; +- a **worker** per session (name `session:S1` for session S1): runs the agent and works on the session's files; +- **clients**: browsers (the web UI) and TUIs (a terminal UI), any number. + +**How they connect:** + +``` +browser ──┐ WebSocket IPC ┌── worker session:S1 +browser ──┼───────────── server ─────┤ +TUI ──────┘ └── worker session:S2 +``` + +- Each connection is a **link**. Clients connect only to the server, over a WebSocket. The server starts a worker as a + child process when its session is first needed and talks to it over IPC (the pipe between a parent process and its + child). +- Clients never talk to a worker directly; the server **relays**. A browser call to something in session S1 travels + browser → server → worker `session:S1`, and the answer comes back the same way. Workers reach each other through the + server too. +- When a client connects, the server checks who it is (its cookie) and remembers that for the whole link. Concept 2 + builds on this. + +**Chord** is the runtime under p2: it lets plugins define **objects** that live on one host and lets any host use them +across links. + +**The goal that shapes everything: reload.** You change a plugin's code, and the new code takes over inside the +running app. No host restarts: the server, the workers and the browsers keep running, and so do open terminals, +running scripts, the agent's current answer and connected browsers. Only the changed plugin's code is swapped. This is +a **reload**; the opposite is a **restart**, where a process ends and starts again and everything it held in memory is +lost. + +Much of what follows exists to make reloads safe: what must survive a reload (data, open files), what must be replaced +(behaviour), and how running work is handed over. Concept 11 shows how a reload works step by step; until then it is +enough to know: **on a reload, the plugin's factories run again with the new code, in the same running process.** + +**A plugin** is a folder under `plugins/`. Each file is meant for certain hosts: + +| File | Runs on | Holds | +| --- | --- | --- | +| `contract.ts` | every host that imports it | descriptions only: names, shapes, method signatures | +| `server.ts` | the server | code for the server | +| `worker.ts` | every session worker | code that needs the agent or the session's files | +| `web.tsx` | the browser | UI components | +| `tui.ts` | the TUI | terminal UI | + +The running example is a **notes** plugin: + +- notes per session, stored in SQLite; +- a text file dropped into an inbox folder becomes a note; +- notes older than 30 days are deleted every hour; +- a web panel shows the notes; the owner and invited guests with write access can add notes, read-only guests can only + read. + +--- + +## Concept 1: Object types and objects + +**The problem.** Several places need "the notes of session S1": the owner's browser tab, a guest's tab, an HTTP export +handler. If each built its own copy, they would drift apart, and nobody would know when to clean up. + +**The idea.** There is **one object** "notes of S1". Everyone asks Chord for it by its **address** and gets the same +one. + +- An **object type** describes what such an object is: its name, what its address looks like (the **input**), on + which host it lives, and what you can do with it (its **members**: data you can read and methods you can call). It + contains no code that runs; it is a description every host can share. Another word for it is **contract**. +- A **factory** is the code that builds an object of that type, on the host where it lives. +- An **object** is one concrete instance: "Notes for `{ session: "S1" }`". + +The three pieces live in three files of the notes plugin: + +```ts +// plugins/notes/contract.ts: the object type, imported by the worker and the browser +export const Notes = remote("notes", { // a type usable across links, named "notes" + version: 1, // the contract's version (concept 12) + host: sessionHost, // lives on the worker "session:" + input.session + input: Type.Object({ session: Type.String() }), // the address: { session: "S1" } (a TypeBox schema) + variants: { /* the members, concept 2 */ }, +}); +``` + +```ts +// plugins/notes/worker.ts: the factory, runs on every session worker +export default defineFacet((host) => { // runs when the plugin loads; host = this worker (concept 7) + host.serve(Notes, (input, principal, self) => { // "this host builds Notes objects like this" + // input = { session: "S1" }; principal: concept 2; self: concept 3 + return { /* members */ }; + }); +}); +``` + +```ts +// plugins/notes/web.tsx: a caller, in the browser +const notes = await host.get(Notes, { session: "S1" }); // browser → server → worker session:S1 +``` + +- **Where an object lives** is decided by the type alone. p2 defines two placements (`src/runtime/chord.ts`): + - `sessionHost = hostFamily("session:", "session")`: the host name is the prefix `"session:"` plus the input's + `session` field. For `{ session: "S1" }` that is `session:S1`. The input must have that field; + - `serverHost = hostNamed("server")`: always the host `server`. + + The caller only names the type and the input; Chord computes the host and sends the request there through the + server. +- **Who builds it:** the host where it lives runs the factory. That is why the factory is in `worker.ts`: `server.ts` + could not serve `Notes`, and Chord refuses that at startup. +- **One object per address:** the first time anyone asks for `{ session: "S1" }`, Chord runs the factory. Everyone who + asks later for the same address gets the same object (concept 2 refines this). `{ session: "S2" }` is another object, on another worker. +- **`local` and `remote`:** a `local(...)` type can only be used on its own host; a `remote(...)` type can also be used + from other hosts across links (here: the browser using an object on the worker). + +**Check questions** + +1. Two tabs ask for the notes of S1. How many objects exist, and how often did the factory run? +2. What is the difference between `Notes` and "Notes for S1"? +3. Where does the object for S2 live, and who decides that? + +*Model answers:* (1) one object, the factory ran once (same address; concept 2 refines this when the callers are +different people). (2) `Notes` is the description (type); "Notes for S1" is one instance (object). (3) In the worker of +session S2; the type's `host` declaration decides, derived from the input. + +--- + +## Concept 2: Who is calling (principals and variants) + +**The problem.** The owner may do everything; a read-only guest may only read; a guest of another session may do +nothing. The object must know who calls, and the caller must not be able to lie. + +**The idea.** Every call carries a **principal**, the caller's identity. It is never taken from the message: the link +stamps it, from what the server learned when the client connected (its cookie, see the setting). When the server +relays a call to a worker, it passes the principal along; a worker accepts a passed-along principal only from its +parent link, the server. So nothing a client sends, at any hop, changes who it is. + +p2's principals, each with a `kind`: + +| `kind` | Who | +| --- | --- | +| `owner` | the person running p2 | +| `guest` | someone invited into one session: `{ kind: "guest", session: "S1", mode: "view" \| "full", … }` | +| `worker` | a session worker calling the server | +| `server` | the server calling a worker | +| `host` | a host's own code calling an object on the same host | + +The factory gets the principal and decides what the caller gets. An object type can have **variants**: named sets of +members. The factory returns one of them, named by `variant`. + +```ts +// plugins/notes/contract.ts: the variants of Notes +variants: { + reader: { state: { notes: NotesSchema } }, // can read the notes + editor: { state: { notes: NotesSchema }, methods: { add: method(AddArgs) } }, // can also add one +} +``` + +```ts +// plugins/notes/worker.ts: the factory picks the variant per caller +host.serve(Notes, ({ session }, principal, self) => { + if (!canView(principal, session)) throw forbidden(); // refuse: the caller gets an error, no object + const notes = loadNotes(session); // simplified; see the problem below + if (!canSend(principal, session)) return { variant: "reader", notes }; + return { variant: "editor", notes, add: (args, ctx) => addNote(session, args, ctx) }; +}); +``` + +`canView` and `canSend` are p2 helpers (`src/runtime/principals.ts`): + +- `canView`: the owner, or a guest of this session; +- `canSend`: the owner, or a guest of this session with `mode: "full"`. + +- **A reader has no `add` at all.** Chord refuses calls to members that are not in the caller's variant, so `add` needs + no check of its own. +- **The caller sees which variant it got.** A reference is typed as one of the variants, told apart by `variant`. + TypeScript lets you use `add` only after checking `notes.variant === "editor"`; at runtime, using a member of another + variant throws `not_in_variant`: + + ```ts + // plugins/notes/web.tsx: a caller + const notes = await host.get(Notes, { session: "S1" }); + notes.add({ text: "hi" }); // type error: `add` exists only on the editor variant + if (notes.variant === "editor") { + await notes.add({ text: "hi" }); // fine: narrowed to editor + } + ``` +- **One object per (type, input, caller).** The factory builds something different per caller, so each caller gets its + own object. Concept 1's "one object per address" holds per caller: two tabs of the owner share one `Notes`; two guests + get one each. + +**The problem this creates.** If each caller has its own `Notes`, and the notes were loaded inside it, there would be +one copy per caller again: exactly what concept 1 wanted to avoid. + +**Store plus view.** The data goes into a second object type that everyone shares, and the per-caller object only checks +and forwards: + +```ts +// plugins/notes/worker.ts: a local type, defined where it is used +const NoteStore = local("notes.store", { + input: Type.Object({ session: Type.String() }), + members: { notes: …, add: method(AddArgs) }, +}); +host.serve(NoteStore, ({ session }, principal, self) => ({ /* loads and holds the notes of one session */ })); + +host.serve(Notes, ({ session }, principal, self) => { + if (!canView(principal, session)) throw forbidden(); + const store = self.get(NoteStore, { session }); // get another object from inside a factory (concept 3) + if (!canSend(principal, session)) return { variant: "reader", notes: store.notes }; + return { variant: "editor", notes: store.notes, add: (args, ctx) => store.add(args, ctx) }; +}); +``` + +- `NoteStore` is `local`: only code on this worker can get it, so no browser ever reaches it directly. +- Every `Notes` factory asks for it with the same principal, `host` (this worker's own code), so there is one + `NoteStore` per session, shared by all `Notes` objects of that session. +- This "store plus view" pattern is everywhere in p2. + + +**Check questions** + +1. A read-only guest calls `add`. What happens, and where is it decided? +2. Could a malicious browser claim to be the owner? Why not? +3. The owner has a tab open, two guests are in S1. How many `Notes` and how many `NoteStore` objects exist? + +*Model answers:* (1) The guest got the `reader` variant (the factory decided), which has no `add`; Chord refuses the +call. (2) No: the link stamps the principal from the server's authentication of the connection, never from a message; +when relayed, the worker accepts it only from the server. (3) Three `Notes` (owner, guest A, guest B; a second owner tab +would share the owner's), one `NoteStore` (every `Notes` gets it as `host`). + +--- + +## Concept 3: References + +**The problem.** When can an object be cleaned up? Not while someone uses it, but not never either. + +**The idea.** Asking for an object gives you a **reference**, not the object. While at least one reference exists, the +object lives. When the last one is released, Chord cleans it up (and the object releases the references it holds +itself). + +There are three ways to get a reference; they differ in when it is released: + +| Where you are | How | Released when | +| --- | --- | --- | +| in a factory | `self.get(Type, input)` | the object holding it ends | +| in a method | `await ctx.get(Type, input)` | that call ends | +| outside Chord (startup code, UI) | `await host.get(Type, input)` | you call `ref.release()` | + +`ctx` is the context every method call gets as its last argument (concept 10); here it only matters that `ctx.get` +binds a reference to one call. + +**Which one:** ask how long the object needs the other object. + +```ts +// plugins/notes/worker.ts +host.serve(Notes, ({ session }, principal, self) => { + const store = self.get(NoteStore, { session }); // needed for the object's whole life: factory, self.get + return { + variant: "editor", + notes: store.notes, + add: async (args, ctx) => { + const filter = await ctx.get(SpamFilter, {}); // needed for this call only: method, ctx.get + if (await filter.check(args.text, ctx)) await store.add(args, ctx); + }, + }; +}); +``` + +`self.get` belongs in the factory only: it declares what the object keeps. Called in a method, it would take another +long-lived reference on every call. + +**A trace:** + +1. The owner opens the notes panel; the browser calls `host.get(Notes, { session: "S1" })`. The worker builds the + owner's `Notes`; its factory's `self.get` builds `NoteStore`. Chain: browser → owner's `Notes` → `NoteStore`. +2. A guest joins and gets their own `Notes`, which references the same `NoteStore`. +3. The owner closes the tab. The browser's reference is released, the owner's `Notes` ends and releases its reference + to `NoteStore`. The guest's `Notes` still holds it: the store stays. +4. The guest closes too. The guest's `Notes` ends, `NoteStore` has no reference left and ends. + +A lost link counts as closing: when a link dies, everything held through it is released. + +**More rules:** + +- **A reference you hold is usable:** the object is live and its data is there, whether it lives on this host or + another one. +- `self.get` in a factory is synchronous: it only declares. Chord makes the object live after the objects it references + are ready. +- `host.serve(Type, factory, { linger: 30_000 })` keeps an unreferenced object 30 seconds longer: closing and reopening + the panel quickly rebuilds nothing. +- `host.serve(Type, factory, { lifetime: "host" })` makes the object live as long as its host, referenced or not. It is + created when the plugin loads, not at the first `get`. Such a type is `local` and has no required input, so there is + exactly one per host (each worker has its own). For services every plugin needs. + +**Check questions** + +1. The owner closes the tab; a guest still has it open. Does `NoteStore` end? And when the guest closes too? +2. `Notes.add` checks every note against `SpamFilter`. (a) `add` is rare and the filter is expensive to keep alive; + (b) every `Notes` uses the filter constantly. Where and how do you get the reference in each case? +3. What does `lifetime: "host"` change? + +*Model answers:* (1) No, the guest's `Notes` still references it; when the guest closes too, it ends (after `linger` +if set). (2) (a) `ctx.get` in `add`: alive only while a call needs it; (b) `self.get` in the factory: alive as long as +the `Notes`, no call waits for it to start. `self.get` never goes into a method. (3) The object exists from loading +until its host stops, independent of references; one per host. + +--- + +## Concept 4: State + +**Where we are.** The notes plugin so far: + +```ts +// plugins/notes/contract.ts +export const NoteSchema = Type.Object({ id: Type.Number(), text: Type.String() }); +export const NotesSchema = Type.Array(NoteSchema); +export const AddArgs = Type.Object({ text: Type.String() }); + +export const Notes = remote("notes", { + version: 1, + host: sessionHost, + input: Type.Object({ session: Type.String() }), + variants: { + reader: { state: { notes: NotesSchema } }, + editor: { state: { notes: NotesSchema }, methods: { add: method(AddArgs) } }, + }, +}); +``` + +```ts +// plugins/notes/worker.ts +const NoteStore = local("notes.store", { input: Type.Object({ session: Type.String() }), members: { /* notes, add */ } }); + +export default defineFacet((host) => { + host.serve(NoteStore, ({ session }, principal, self) => ({ /* holds the notes: this concept */ })); + + host.serve(Notes, ({ session }, principal, self) => { + if (!canView(principal, session)) throw forbidden(); + const store = self.get(NoteStore, { session }); + if (!canSend(principal, session)) return { variant: "reader", notes: store.notes }; + return { variant: "editor", notes: store.notes, add: (args, ctx) => store.add(args, ctx) }; + }); +}); +``` + +Two things are missing: what `notes` is inside `NoteStore`, and how a browser shows it and stays current. + +**The problem.** The notes change: someone adds one. Every open panel, on any host, should show the change at once. A +plain variable can't do that: + +- nobody is told when it changes; +- it can't travel over a link; +- the factory runs again on every reload (see the setting), so a variable created there starts empty again. + +**The idea.** **State** is a named, observable value with a schema (a **cell**). The object that declares it owns it and +is the only one that writes it. Anyone holding a reference to the object can read it and is told when it changes, on +any host. + +### Declaring and writing (the owning object) + +For now the notes live only in memory: + +```ts +// plugins/notes/worker.ts +host.serve(NoteStore, ({ session }, principal, self) => { + const notes = self.state("notes", NotesSchema, { init: () => [] }); // name, schema, first value + + return { + notes, // returned: holders of a reference can read and watch it + add: async ({ text }, ctx) => { + const id = (notes.value.at(-1)?.id ?? 0) + 1; + notes.change(ctx, (draft) => { + draft.push({ id, text }); + }); + }, + }; +}); +``` + +- `self.state(name, schema, { init })` declares the cell. `init` gives its first value, here an empty list; it runs + before the object goes live. (Concept 5 loads the first value from a database instead.) +- **Writing:** `notes.change(ctx, (draft) => …)` edits a draft: atomic (all or nothing if the callback throws), and only + the paths you touch change. `notes.replace(ctx, value)` sets the whole value. +- **Every write needs a `ctx`**, so writes happen only inside calls. The factory gets no `ctx`: it only declares. +- `notes.value` is immutable: never mutate what you read. +- What the factory returns is visible to holders of a reference; a cell it doesn't return is private. + +### Passing state on (the view) + +`Notes` returns `notes: store.notes`: the store's cell becomes a member of every `Notes` object. It is not a copy: all +`Notes` of S1 show the same value, with no code to keep them in sync. + +### Reading and watching (the consumers) + +Every state member of a reference has `.value` (the current value) and `.subscribe(fn)` (called on every change; returns +a function that stops it). An awaited reference already has the first value. + +Without UI (any host, a script, a test): + +```ts +const notes = await host.get(Notes, { session: "S1" }); +notes.notes.value; // [{ id: 1, text: "buy milk" }] +const stop = notes.notes.subscribe((list) => console.log(list.length)); +// … +stop(); +notes.release(); +``` + +In the browser, p2's React hooks (`src/runtime/web.tsx`) do the same for a component: + +```tsx +// plugins/notes/web.tsx +function NotesPanel({ session }: { session: string }) { + const notes = useObject(Notes, { session }); // host.get while mounted, release on unmount + const list = useReadable(notes?.notes); // the current value; re-renders on every change + if (!notes || !list) return

Loading…

; + return ( +
+
    {list.map((note) =>
  • {note.text}
    • )}
+ {notes.variant === "editor" && ( // TypeScript knows: `add` exists only for editors + + )} +
+ ); +} +``` + +Inside an object, reacting to state (for example, writing a file whenever the notes change) is not a `subscribe` in the +factory but an effect (concept 6). + +### One add, step by step + +The owner and a full guest both have the panel open. The guest clicks Add: + +1. The guest's browser calls `notes.add({ text: "hi" })`. The server relays it to worker `session:S1`, to the guest's + `Notes` (variant `editor`), which forwards to `store.add`. +2. `store.add` appends the note with `notes.change`. +3. Chord sends the **change** (the one new element, not the whole list) to every host holding a reference with that + member: the owner's browser and the guest's browser, both through the server. +4. In each browser `useReadable` re-renders the panel. + +A new reference gets the whole value once, then only changes; after a reconnect or a reload it gets the whole value +again. The call's result and the state change travel separately: when `await notes.add(…)` returns, the panel may not +show the note yet. Wait for the state, not for the call. + +### Derived data + +```ts +const count = self.derive([notes], (list) => list.length); +return { notes, count, add }; +``` + +A derivation is computed from other state and recomputed when it changes. Nobody writes it, and it needs no name: after +a reload it is simply computed again. + +### Why cells have names + +On a reload the factory runs again; `self.state("notes", …)` then returns the **existing** cell with its current value, +and `init` does not run again. So the notes survive a reload. They do not survive a **restart** (the process ends): the cell lives in +memory. Concept 5 fixes that. + +If the schema or meaning of a cell changes, you raise its `version`; concept 5 shows what then happens. + +**Check questions** + +1. Why can't the factory write state? +2. You reload the notes plugin, then later the worker process restarts. What happens to the notes each time? +3. A guest should see only the number of notes. What do you change, and on which objects? +4. The owner and a guest have the panel open; the guest adds a note. What reaches the owner's browser, and how? + +*Model answers:* (1) The factory only declares and has no `ctx`; it runs again on every reload (first as a trial), so a +write there would overwrite the surviving value or happen in a discarded trial. (2) Reload: kept, the named cell is +returned with its value. Restart: gone, the cell was only in memory. (3) `NoteStore` derives `count` and returns it; +`Notes` gets a variant with only `count` (for example `counter: { state: { count: Type.Number() } }`) and returns +`count: store.count` for such guests. (4) Only the change to `notes` (the new note), sent from the worker through the +server; `useReadable` re-renders. No call and no full list. + +--- + +## Concept 5: Resources + +**Where we are.** `NoteStore` keeps the notes in a cell (concept 4): + +```ts +// plugins/notes/worker.ts +host.serve(NoteStore, ({ session }, principal, self) => { + const notes = self.state("notes", NotesSchema, { init: () => [] }); + return { + notes, + add: async ({ text }, ctx) => { + const id = (notes.value.at(-1)?.id ?? 0) + 1; + notes.change(ctx, (draft) => { draft.push({ id, text }); }); + }, + }; +}); +``` + +**The problem.** A cell lives in memory: when the worker restarts, the notes are gone. They must be stored in a file, +here SQLite. But where do you open the file? + +```ts +// wrong +host.serve(NoteStore, ({ session }, principal, self) => { + const db = openDb(notesPath(session)); // runs on every reload: a new handle each time, the old ones never closed + // … +}); +``` + +The factory runs again on every reload. Each run would open another file handle, and nothing ever closes them, not +even when the object ends. + +**The idea.** A **resource** is a native thing the object opens once and closes once: a database handle, a socket, an +SSH connection, a sandbox. It is declared by name, like a cell, and survives reloads. + +```ts +const db = self.resource("db", { version: 1 }, + () => openDb(notesPath(session)), // open: runs once, before the object goes live + (db) => db.close()); // close: runs once, when the object ends +db.value; // the open handle (in methods) +``` + +**The store, now persistent:** + +```ts +// plugins/notes/worker.ts +host.serve(NoteStore, ({ session }, principal, self) => { + const db = self.resource("db", { version: 1 }, () => openDb(notesPath(session)), (db) => db.close()); + const notes = self.state("notes", NotesSchema, { + version: 2, // was: 1 (the default); see "Reloading into this version" + migrate: (old) => old, // see "Reloading into this version" + init: async ({ previous }) => { // was: () => [] + if (previous) db.value.insertMissing(previous); // only on the reload from concept 4 + return db.value.all(); + }, + }); + + return { + notes, + add: async ({ text }, ctx) => { + const id = db.value.insert(text); // 1. the truth: SQLite + notes.change(ctx, (draft) => { draft.push({ id, text }); }); // 2. the live copy + }, + }; +}); +``` + +What changed: + +- `db` is a resource: opened before the object goes live, closed when it ends. +- `init` no longer returns `[]` but loads the notes from SQLite **before** the object goes live, so no caller ever + sees an empty list that fills a moment later. It runs when the object is built (first use, after a restart). On a + reload a kept cell keeps its value and `init` does not run. +- `add` writes SQLite first, then the cell. SQLite is the truth; the cell is the live copy others watch. If the + process dies between the two, the note is still in SQLite, and `init` loads it next time. +- The ids now come from SQLite. +- The cell has `version: 2`, a `migrate` and an `init` that takes `previous`. That is for one moment only: the reload + from concept 4's code to this one. + +**Reloading into this version.** The app is running the in-memory store from concept 4 and holds a few notes. You +reload into this code. The cell `notes` keeps its name and schema, but its **meaning** changed: before, the cell *was* +the data; now it is a copy of SQLite. + +Without a bump, the reload keeps the cell with its value, and `init` does not run. The new code would serve the +in-memory notes as if they came from SQLite, but they are not in SQLite: the next restart loses them. Chord cannot +notice this: same name, same schema, only the code inside `init` changed. **Raising the version is your job.** + +With `version: 2`, the reload treats the cell as changed, in two steps: + +1. **`migrate(old)`** converts the old value to the new schema. It must be pure: it runs once as a trial while the + reload is checked, and once for real. The schema did not change, so here it returns `old` unchanged. Without a + `migrate` (or if it returns `undefined`), there is no `previous`, and the old notes are dropped. +2. **`init({ previous })`** then runs once, with the migrated value as `previous`. Unlike `migrate` it may write: + here it inserts the in-memory notes into SQLite, then loads everything. It must be safe to run twice (it reruns if + the reload is retried), hence `insertMissing`. + +Meanwhile `NoteStore` is handed over (concept 11): running calls finish first and new calls wait, so no `add` runs +while `init` copies; then `db` is open, `init` runs, the waiting calls continue, and browsers get the whole value +again. After a restart `previous` is absent and `init` only loads. + +The rule: **when the schema or the meaning of a cell changes, raise its version.** Chord refuses a schema change +without a bump; a meaning change (a different `init`) it cannot see. + +**Three kinds of cells.** Cells are the live view, not the store. Most cells are one of these: + +| Kind | Example | `init` | On a version bump | +| --- | --- | --- | --- | +| mirrors a source (SQLite, Durable) | `notes` from now on | reads the source, ignores `previous` | nothing to carry over: no `migrate`; bump freely | +| exists only in memory | concept 4's `notes`, drafts, UI state | `previous ?? fallback` | `migrate` converts the old value | +| large data | a transcript | loads a bounded window (the latest entries) | as a mirror | + +- The `notes` cell changed kind in this concept: from memory-only to mirror. The `migrate` and the `previous` branch in + `init` exist only for that one transition; once every running worker has reloaded past it, they can go, and the + cell is a plain mirror: `init: () => db.value.all()`. +- Large data stays in its source. The cell holds only a window, kept current by an effect (concept 6), and methods + return older pages on request (p2's chat has a `window` cell and a `history` method). Clients cache the pages they + fetched. + +**Resource rules:** + +- On a reload, the same name and `version` keep the resource open; the new code uses the same handle. +- Chord closes resources when the object ends, in reverse order of declaration. +- **Only native values, no behaviour.** An event handler or timer set up inside `open` would keep running the old code + after a reload, because `open` doesn't run again. Behaviour goes into effects (concept 6). +- Raise `version` when what `open` produces changes (another file, other options). Two kinds: + - `parallel` (default): old and new can exist side by side; the new one opens first, the old one closes once nothing + uses it anymore; + - `exclusive` (a port, a lock): the old one must close before the new one opens (concept 11). + +**Check questions** + +1. Why not open the DB directly in the factory? +2. You reload from the in-memory store to this one but forget `version: 2`. What happens to the notes, now and at the + next restart? Why doesn't Chord stop you? +3. The worker restarts. Walk through what happens to the notes until a browser sees them again. +4. You change a query inside a method. Do you raise the resource's `version`? And if you change the file name? +5. Someone put a file-change handler inside `open`. What happens after you fix that handler and reload? + +*Model answers:* (1) The factory runs on every reload; each run would open another handle, never closed. (2) The reload +keeps the cell with the in-memory notes; `init` doesn't run, so they never reach SQLite, and the next restart loses +them. Chord sees the same name and schema; a changed `init` is invisible to it, so the bump is the author's job. (3) The cell +is gone with the process; the notes are in SQLite. When a browser asks again, `NoteStore` is built: `open` opens the +file, `init` loads the notes, then the object goes live and the browser gets the whole value. (4) No for the query +(method code is new anyway); yes for the file name (what `open` produces changes). (5) Nothing: `open` doesn't run +again, the old handler keeps running; that's why behaviour belongs in effects. + +--- + +## Concept 6: Effects and operations + +**The problem.** An object must react (an inbox file appears), run background work (hourly cleanup), or keep a long +task going (an OAuth login that takes minutes). Done directly in the factory, every reload would add another handler +or timer. + +**The idea.** An **effect** is behaviour Chord starts and stops for you: + +```ts +self.effect(onEvents(watchDir(inbox), async (file, ctx) => { + db.value.insert(await readFile(file, { signal: ctx.signal })); + notes.replace(ctx, db.value.all()); +})); +self.effect(every("cleanup", HOUR, () => db.value.deleteOld())); +``` + +- An effect starts when the object is live and stops when it ends. +- When the object's **code changes**, the old effect is stopped first, then the new one starts, so they never run side + by side. Handlers already running finish. +- A handler on **state** gets the current value again after a restart, so it must **reconcile** ("make the world match + this value"), not blindly act. +- A named timer (`every("cleanup", …)`) keeps its next deadline across reloads. + +An **operation** is long-running work an object owns, like a login flow: + +```ts +const flow = self.operation(async (ctx) => runOAuth(provider, ctx)); +``` + +It keeps its own code until it finishes, so a reload of unrelated code doesn't interrupt it. + + +**Check questions** + +1. Why does a handler on state have to reconcile? +2. You change the cleanup interval's callback and reload. Does the hourly schedule restart from now? +3. Effect or operation for "send a daily report"? For "an OAuth login in progress"? + +*Model answers:* (1) It receives the current value again after a restart; a blind action would run twice. (2) No, the +named timer keeps its deadline; only the callback changes. (3) Daily report: an effect with a named `every`; OAuth +login: an operation (long-lived, must not be interrupted by an unrelated reload). + +--- + +## Concept 7: The facet and registries + +**The problem.** A plugin must contribute things to the app it can't edit: an HTTP route, a tool for the agent, a panel +in the UI. And the plugin's objects need to share things like the DB. + +**The idea.** + +- A **facet** is a plugin's code for one process (its `server.ts`, `worker.ts`, `web.tsx` or `tui.ts`). The facet's + setup is itself an object that lives as long as the process: it can declare resources and state, and its served + objects share them. +- A **registry** is a named table someone owns (the app or a plugin) and others contribute to: + +```ts +export default defineFacet((host, self) => { + const db = self.resource("db", { version: 1 }, () => openDb(host.value(DataDir)), (db) => db.close()); + self.register(Routes, { "GET /notes/export": (req, res, access, ctx) => exportNotes(db.value, res) }); + host.serve(NoteStore, (input, principal, self) => ({ /* uses db */ })); +}); +``` + +- The owner declares the registry and reads it (`host.registry(Routes).table()`); plugins register entries. +- Entries live as long as the plugin is loaded; a reload swaps the whole table at once, so there is never a moment with + a missing route. +- Registries are per process: a worker facet can't add a server route. Things that cross processes go through objects. +- Only the facet and `lifetime: "host"` objects register (an object that comes and goes would make its entries flicker). + + +**Check questions** + +1. Why can't an ordinary (referenced) object register a route? +2. Your worker facet wants an HTTP route. How? +3. What happens to a plugin's routes when the plugin is removed? + +*Model answers:* (1) Its entries would exist only while someone references it: the route would flicker. (2) The server +facet registers the route; the handler calls an object in the worker. (3) They disappear with the plugin's facet. + +--- + +## Concept 8: Host values + +**The problem.** Some things exist once per process and belong to the app, not to any plugin: the data folder, the +agent runtime (Durable harness), the session's current directory. + +**The idea.** The app provides them as **host values**, typed by a key: + +```ts +const dir = host.value(DataDir); // a string +const harness = host.value(Harness); // the Durable harness +``` + +- They are there before any plugin loads; no waiting, no `!`. +- Each key has a version; a plugin built against an older version gets an adapted value or a clear error at load. + + +**Check questions** + +1. Why are host values not ordinary objects? +2. What happens when a plugin asks for a key the process doesn't provide? + +*Model answers:* (1) They are native, process-wide things owned by the app, not data or methods, and must exist before +plugins load. (2) It fails with a clear error when the plugin loads. + +--- + +## Concept 9: Who is here (`host.references`) + +**The problem.** The owner wants to see which guests are in a session; the worker should stay running while someone +looks at it. + +**The idea.** Chord knows which references exist, so it can tell you: + +```ts +const viewers = self.derive([host.references(Chat)], (refs) => refs.length); +``` + +`host.references(Type)` lists who (input, principal, how many clients) holds references to objects of a type in this +process. A disconnect counts as leaving. + + +**Check questions** + +1. Why is this better than each client registering itself in a list? + +*Model answer:* Chord already tracks references and updates them on connect, release, revocation and disconnect; no +extra objects, no enter/leave races on reload. + +--- + +## Concept 10: `ctx` and call trees + +**The idea.** Every method, handler and operation gets a `ctx`: + +- `ctx.signal`: fires when the call is cancelled (the caller gave up, the object ended, or a reload needs it to stop); +- `ctx.get`: references that live for this call; +- the process's own principal. + +A **call tree** is everything one starting point causes (a click in the browser, an event), across objects and +processes. + +**The rule:** pass `ctx` or `ctx.signal` to everything you wait on (`fetch(url, { signal: ctx.signal })`). Then a long +call can be stopped cleanly. + +**Check questions** + +1. A method waits on `fetch` without the signal. What goes wrong when it must be stopped? + +*Model answer:* It can't be interrupted; it keeps running until the fetch returns, and anything that has to wait for it +(a reload's handover) waits longer. + +--- + +## Concept 11: Reload + +**The problem.** You change a plugin's code. Everything should keep running: open terminals, running scripts, the +agent's current answer, connected browsers, without restarting any process. + +**The idea, in one sentence:** behaviour is replaced at once; storage is handed over. + +On each process: + +1. **Prepare:** the new code is loaded and checked: factories run as a trial (no side effects), contracts are checked + against every current caller, new `parallel` resources open. If anything fails, nothing changes. +2. **Publish:** in one step, every call from now on reaches the new code. Objects whose storage changes (a state + version bump, an `exclusive` resource) are fenced in that same step: new calls to them wait. All other objects are + done: their effects restart with the new code; calls already running finish their old code. An object that was + still being built at that moment starts over with the new code. + + Objects that merely hold a reference to a fenced object are **not** fenced: their calls run, and wait only when they + reach the fenced object. (Fencing them would deadlock: their running call waits for the handover, which would wait + for them to finish.) +3. **Hand over** each fenced object: + 1. its running calls, operations and handlers finish (after a deadline they are asked to stop via `ctx.signal`; + nothing is closed while they still run; a call that re-enters the object from inside is let through); + 2. bumped cells get their new value in a staged copy: `migrate` converts the old value (it already ran once in + prepare as a trial, so it must be pure), then `init({ previous })` runs once with the result and may do I/O + (concept 5). `exclusive` resources are closed and reopened; + 3. on success the new state, code and effects are installed and the waiting calls continue. + +If a handover fails (a port is busy, a migration throws), **only that object** fails; you fix the code and reload, and +it comes back in place. + +What callers see: usually nothing; during a handover, a short wait; a call running on that object past the deadline is +cancelled. + +**Example:** you add `tags` to the note schema (and a column to the SQLite table) and raise the cell to version 3. No +`migrate` is needed: the truth is in SQLite. The `NoteStore` objects are handed over: their calls finish, `init` loads +the notes with their tags, and the waiting calls continue. Browsers get the new value. Nothing else stops. + + +**Check questions** + +1. You fix a bug in a method and reload. What happens to a call to that method that is running right now? +2. You raise the version of a sandbox resource (parallel) while a script runs in the old sandbox. What happens? +3. Why can old code never write state that was just migrated? + +*Model answers:* (1) It finishes its old code; new calls get the fix. (2) The new sandbox opens; new scripts use it; +the running script keeps the old one, which closes when the script ends. (3) The object's calls are drained before the +migration, and new calls wait until it is done. + +--- + +## Concept 12: Versions and adapters + +**The problem.** A plugin written for an older p2 should keep working when p2 changes. And a plugin's own state may +change shape over time. + +**Two kinds of versions:** + +- **Interface versions**, between parties that change independently: object types, registries, host values. The + consumer brings the version it was built against; the owner provides an **adapter** for old versions; anything + else fails with a clear error when the plugin loads. + +```ts +host.serve(Notes, factory, { accepts: { 1: (current) => oldView(current) } }); // old callers keep working +``` + +- **Internal versions**, between old and new code of the same object: state and resource versions. Nobody adapts + them; a bump triggers a handover (concept 11) with `migrate` or a reopen. + + +**Check questions** + +1. A plugin was built against `Notes` version 1; p2 now serves version 2. When does it keep working? +2. Why don't state versions get adapters? + +*Model answers:* (1) If version 2's provider declares an adapter for version 1 (or the change was compatible and the +version didn't change); otherwise it fails at load. (2) They aren't shared with anyone else: only the object's own old +and new code; a migration converts the value once. + +--- + +## Glossary + +| Term | Meaning | +| --- | --- | +| facet | a plugin's code for one process; itself a long-lived object | +| object type | the description of objects: name, input, members, version, host | +| object | one instance per (type, input, caller) | +| principal | who calls, stamped by the server | +| variant | the set of members a caller gets | +| reference | what `get` returns; keeps the object alive | +| state | named, observable data; survives reloads | +| resource | a named native thing; survives reloads | +| effect | behaviour restarted when code changes | +| operation | long-running owned work | +| registry | a declared, versioned table of contributions | +| host value | a native value the app provides | +| handover | replacing an object's storage in place during a reload | diff --git a/apps/p2/docs/durable-request.md b/apps/p2/docs/durable-request.md new file mode 100644 index 000000000..efa2f61b7 --- /dev/null +++ b/apps/p2/docs/durable-request.md @@ -0,0 +1,52 @@ +# Request to Durable: share the document transaction core + +**Superseded:** p2 builds its own document store in Chord with a storage interface and adapts Durable's `Storage` in p2 +(`target-design.md` §9). Only moving `OutputBuffer`/`Progress` to Chord is still under consideration. + +From the Chord/p2 redesign (`target-design.md`). To be implemented on pi `main` (`packages/durable`, plus `packages/chord` +where noted), then merged into `private`. + +## Why + +p2 moves object state into transactional documents: drafts that record Chord ops, version plus `migrate` on read, +`checkpointWhen`, create and retire, one atomic storage batch per commit, adopt and publish afterwards. Durable's +`Transaction` already does exactly this and is battle-tested. p2 must reuse it instead of writing a second +implementation. Chord itself must stay free of a Durable dependency; p2 already depends on Durable and builds its object +store on top of it. + +## What we need + +1. **A document-only transaction core**, exported from Durable (for example `@earendil-works/pi-durable/documents`): + - acquire a draft for a document address (`kind`, `scope`, optional `key`), creating it on first write + (`initial()`), loading and migrating it through the host otherwise; + - retire a document; + - prepare every draft at settlement (revoking the drafts), plan the storage writes (`document.create` / + `document.change` / `document.retire`, base or delta per `checkpointWhen`), discard on failure or abort; + - adopt after the storage commit and return the publications (record, version, ops) so the caller can notify; + - the existing rules: sealing (`Transaction has settled`), read-after-write, version mismatch handling (a token whose + version differs from the cached value reloads; a stored version newer than the token is refused). + + Durable's `Transaction` then builds on this core and adds tasks, conversations, submissions and forks, so the core is + covered by Durable's existing tests. The seam already exists: `TransactionHost` (`session/transaction.ts:99`); + `conversationCreated` is its only Durable-specific member. +2. **Definitions usable outside a conversation:** `defineDoc` with `scope: "session"` already exists; the core needs to + accept a session-scoped address with an arbitrary `kind` and `key` chosen by p2 (p2 maps an object's identity hash, + the state name and the key into them). Confirm whether `kind` must be globally registered or may be dynamic. +3. **Move `OutputBuffer` and `Progress`** (`harness/output.ts`) to Chord (for example `@earendil-works/chord/progress`); + Durable imports them from there and **re-exports** them under their current names, so nothing breaks. +4. **Review of p2's use of `Storage`:** p2's store calls the core and `Storage` directly (memory and SQLite backends), + without a `Session`. It must uphold what `Session` guarantees to `Storage`: serialized commits (p2 serializes per + host), create before change, no change after retire, ids from `mintId`, deltas against the stored version. We would + like the Durable owners to confirm the list is complete. + +## What p2 does with it (for context) + +- One store per host. Documents are owned by one object identity; the mutation line is per object; storage commits are + serialized per host. In-memory documents use the memory backend semantics without persistence; `persist: true` + documents go to the host's SQLite storage. +- Commits notify local subscribers once and send one atomic `state` batch per hold on the wire. +- No eviction: a document's memory copy lives as long as its object; persisted documents stay on disk until retired. + +## Out of scope + +Tasks, conversations, submissions, forks, the harness and the scheduler stay as they are. diff --git a/apps/p2/docs/handoff.md b/apps/p2/docs/handoff.md new file mode 100644 index 000000000..6e56d55f7 --- /dev/null +++ b/apps/p2/docs/handoff.md @@ -0,0 +1,176 @@ +# p2 handoff + +For whoever continues p2, human or agent, starting cold. It assumes you know pi and this repository, but not the +history of p2. Everything you need is in the repository; nothing depends on earlier conversations. + +## What p2 is + +p2 (`apps/p2`) is Pi Mobile rebuilt on **Chord**, an object runtime (Chord v2: `@earendil-works/chord`, +`packages/chord`; p2 runs on Chord v2 only, see `docs/p2-notes.md` **Chord v2**; Chord v1 is deleted). +Every feature is a plugin (`apps/p2/plugins/*`) with facets per process: `server.ts`, `worker.ts` (one worker per +session, running Durable), `web.tsx` and `tui.ts`. There is no manifest and no dependency list: p2 discovers plugins +by listing their facet files and loads them alphabetically. Chord resolves dependencies by contract name, so their +order does not matter; a missing provider fails loudly, naming the consumer. Compatible backend reloads preserve +unchanged state and resources; a successful reload publishes rebuilt web and TUI bundles. + +p2 replaced Pi Mobile (formerly `apps/mobile`, removed; see git history). Its web UI matches Pi Mobile's (same DOM, +classes, styles); the data layer is Chord objects instead of Pi Mobile's hand-written protocol. On top of Pi Mobile, +p2 has hot reload with explicit reload/restart controls, logs and owner notices, chat mode with replies and `@pi` +mentions, owner sign-in links, an isolated TUI with pi's look and keys, and security hardening (CSP, restricted guest +web assets, WebSocket limits). + +p2 lives in `earendil-works/pi` at `apps/p2` and keeps the name p2. + +## State at handoff + +| Where | What | Port | Data | +| --- | --- | --- | --- | +| Phone (Termux, Node 26) | p2, the daily driver; Pi Mobile's sessions were imported when it replaced Pi Mobile | 8787 | `~/.pi/p2` | +| slayer (`slayer.marioslab.io`) | p2 in dev mode, public at https://p2.mariozechner.at | 8788 | `~/.pi/p2` (test data) | + +Done: everything Pi Mobile did (checked by a four-part audit; what differs on purpose is under Decisions in +`docs/p2-notes.md`), the switch of the phone, the TUI port phases P0–P7 and P9–P11, and Chord in `packages/chord` +replacing the old facet/service runtime and the `pi-client`, `pi-protocol` and `pi-server` packages (pi +`6db4070f1`). Open items are under [Open work](#open-work). + +## The phone + +- **Checkout:** `~/pi-private`, branch `private`. Old data: `~/.pi/mobile` (keep it: imported history refers to + attachments there), `~/.pi/p2.parallel` and `~/.pi/p2.trial-oct10` (leftovers, deletable). +- **p2** runs in tmux session `p2` from `~/pi-private/apps/p2` (as planned for the switch; confirm with `tmux ls` + and `ps`): + ```sh + PORT=8787 P2_ALLOWED_HOSTS=pim.ngrok.pro P2_PUBLIC_URL=https://pim.ngrok.pro npm run dev + ``` + Update: `git pull`, `npm install --ignore-scripts` at the root if dependencies changed, then **Restart** in the app or + `npm run reload -- --restart` (plugin changes: **Reload**). +- **Tunnel:** ngrok as a termux-services (runit) service, `$PREFIX/var/service/ngrok/run`, forwarding + `https://pim.ngrok.pro` to port 8787 (`sv status|restart $PREFIX/var/service/ngrok`; logs in + `$PREFIX/var/log/ngrok/current`). +- **Radius:** not configured (no `radius.json`); `npm run radius:login` enables it. +- **No boot scripts.** Run `termux-wake-lock` so Android keeps the processes alive. + +## Running p2 + +```sh +cd apps/p2 +npm run dev # dev mode: apply plugin changes, including web facets, with Reload or `npm run reload` +npm start # without the dev supervisor +P2_FAKE=1 npm start # fake model, no provider needed +npm run owner-link [url] # one-time owner sign-in link for another device (run on the machine itself) +npm run tui [url] # TUI (Node >= 26.5.1); on loopback it signs in as owner automatically +npm run reload [plugins] [--restart] +npm run check # Biome + tsc +npm test # full suite; TUI tests need Node 26, browser tests need CHROMIUM_BIN +``` + +Environment variables: `docs/p2-notes.md` (Running). Configuration in the data dir: `models.json`, `settings.json` +(incl. `codemode`, default on in p2), `envs.json`, `auth.json`, `radius.json`, `forwards.json`. + +**Sign-in.** Requests from the device itself are the owner. Other devices get owner rights with a link from `npm +run owner-link` (single use, 10 minutes, long-lived signed cookie). Guests join through share links. Requests other +websites make never carry owner rights. Sign out per device or all devices in the Devices sheet. + +**Dev mode.** A file watcher only marks plugins dirty; reloads are explicit (button, `npm run reload`, the +owner-only `reload_plugins` tool) because a watcher can't tell when an agent has finished a multi-file edit. +Runtime code (`src/hosts`, `src/runtime`, workspace packages such as `packages/chord`) needs a restart; the "Server +code changed" pill appears only when a module the server actually loaded changed. + +**slayer.** `~/p2.mariozechner.at/` holds `run.sh` (starts `npm run dev` from the deploy worktree `~/p2-live`), +`deploy.sh` (moves `~/p2-live` to `origin/private` and restarts), `bridge.mjs` (unix socket for Caddy → +127.0.0.1:8788), `Caddyfile` and `docker-compose.yml` (Caddy only marks requests as proxied; p2 authenticates on its +own). It runs in tmux session `p2`. Development happens in `~/pi-private`; `~/p2-verify` is a scratch worktree for +verifying an agent's files on their own before committing. Node 24: `~/node-v24.4.1-linux-x64/bin`; Node 26 (TUI): +`~/.local/bin/node26`. Browser tests: `CHROMIUM_BIN=/home/badlogic/.local/bin/chromium-docker`. + +## Architecture in five minutes + +- **Chord v2** (`packages/chord`, `docs/design.md` there): objects per (type, params, principal) placed on host kinds + (`Server`, `Session({ id })`), JSON Schema contracts, held references, documents (`defineState`) and derives, + natives (`self.open`), keyed registries, binary links over WebSocket and IPC with routes (browser → server → + worker), the workers component (`createWorkers`) and the plugins component (`createPlugins`: build, load, + rollout). How p2 uses it: `docs/p2-notes.md` **Chord v2**. +- **Hosts** (`src/hosts`): `server.ts` (HTTP, auth, WebSocket admission), `web.ts` (web serving), `bundles.ts` + (content-named web and TUI esbuild bundles), `worker.ts` (session worker), + `tui*.ts` (launcher, sandbox, filter, grants), `dev*.ts` (dev supervisor). +- **Runtime** (`src/runtime`): shared contracts and helpers (principals, worker and Harness, Durable registry, + extension switches, input entries, settings, logs, fetch). +- **Plugins**: chat, chat-mode, sessions, subagents, agent (one-shot agents), codemode, coding-tools, prompt, + models, login, search, status, drafts, files, send-file, artifacts, workspace, file-index, terminals, envs, ports, + radius, sharing, devices, proxy-marker, logs, dev-tools, powershell, web-shell. Before writing one, read Chord's + tutorial (`packages/chord/docs/tutorial/`) and plugin guide (`packages/chord/docs/plugin-authoring.md`), then an + existing p2 plugin of the same shape. +- **Locations:** every conversation works at `{ env, cwd }`: the main one at the session's, subagents spawned with a + cwd at their own (on the caller's environment), the others following the session. A session's location changes in + place: relocating an idle running worker fences it, updates the catalog and moves the main conversation and its + followers without a restart; subagents with their own location stay. Plugins resolve a conversation's location + and environment through `Harness.environments`, the session's through `WorkerProcess.location` + (`docs/p2-notes.md`, Agent locations and Relocation). +- **Durable** (`packages/durable`) runs the agent: conversations, inputs as custom entries (`p2.input`, + `chat.mention`), display-only writes placed immediately, code mode (QuickJS sandbox; scripts call tools as + `tools.`, one-shot agents via `tools.agent`), compaction with visible failures, deferred tools and + `tool_search` (unused until MCP lands). + +## The TUI + +`npm run tui [url]` (Node ≥ 26.5.1). The launcher owns the terminal, the connection and sign-in; all TUI code (pi-tui +and plugin facets) comes from the server as one bundle and runs in a worker that drops every Node permission first. +Output passes a strict escape-sequence allowlist; privileged actions (copy, open URL, download, external editor, +attach, saving preferences, sign-out) need launcher grants, mostly with a launcher-drawn consent screen. Preferences +(keybindings, theme, display settings) are the visitor's own (`~/.pi/agent`, `~/.pi/p2-tui/prefs.json`). Threat +model: `DEV.md` and `README.md` (TUI isolation). Design and as-built notes: `docs/p2-tui-port.md`. + +## Rules + +From `AGENTS.md` and this project's history: + +- **One tree.** p2, the packages, root config and scripts change together in `earendil-works/pi`. After p2 changes + run `npm run check` both at the root and in `apps/p2`. +- **Commit only verified states.** When several agents edit one tree, copy an agent's files into a separate worktree + on top of `HEAD`, run `npm run check` and the full `npm test` there, and commit exactly that content + (`git hash-object -w` and `git update-index`). A browser test that times out under load and passes on its own is + acceptable; anything else is not. Lockfile commits need `PI_ALLOW_LOCKFILE_CHANGE=1`; npm 11 strips `libc` fields, + revert that noise. Never `--no-verify`. +- **Deploy only pushed commits** (`~/p2.mariozechner.at/deploy.sh` on slayer). +- **Port from the diff.** Changes copied from elsewhere start from the commit (`git show `), never from a + summary; briefs to agents name the commit and the files. +- **No hand tests on real sessions.** Use throwaway sessions under `/tmp`, or a separate server on another port with + a temporary data dir. Never print cookies, tokens or share links. +- **Check `pwd`** before working in a subagent, and spawn subagents with an explicit `cwd`. +- **`pkill -f` matches its own command line.** Kill by PID or by `/proc//cwd` instead (it once killed the live + tmux session). +- TypeScript: no `any`, erasable syntax only, top-level imports only (except the plugin loader's dynamic import). + Use Opus-class models for documentation and tutorials. + +## Open work + +1. **The phone's daemon binary:** p2's file index uses the pi-env daemon's `listFiles` when the packaged daemon has + it, else the old git/walk index. slayer has a fresh build in `packages/env/bin/pi-env-linux-x64` (static musl; + built with `RUSTFLAGS="-C linker=rust-lld -C link-self-contained=yes" cargo build --release --locked --target + x86_64-unknown-linux-musl` in `packages/env/daemon`). On the phone: `cargo build --release --locked` there and copy + it to `packages/env/bin/pi-env-android-arm64/pi-env`. The next pi-env release ships all targets through CI. +2. **TUI:** launcher images, a filtered terminal relay, a "trust like SSH" mode, web tool cards + (`docs/p2-tui-port.md` §10). Fork from subagent conversations; fork/clone, shell and templates have no web UI yet + beyond rendering. +3. **Cleanup leftovers:** two HTTP helpers were not merged because their semantics differ (byte ranges: files + serves `bytes=-` whole, workspace answers 416; body readers: sharing answers 400 past the limit, devices 413); + decide one behavior each. +4. **MCP:** when it lands in Durable, mark MCP tools deferred and install `tool_search`. +5. **npm:** deprecate `@earendil-works/pi-client`, `pi-protocol` and `pi-server` (1.1.0), which no longer exist. +6. `docs/p2-notes.md` TODO: the TypeScript 5.9.3 pin (Termux), an isolating plugin loader, sandboxing third-party + facets. +7. **Slow test suite:** about 11 minutes serially; files could run in parallel once their isolation is fixed. The + chat-mode long-press reply browser test times out under load. +8. **After the Chord v2 port:** the TUI on Chord's sandbox (C2); no per-link call limit any more (owner decision; + Chord v2 has none). Chord v1 and the old-page shim are deleted (C1). + +## Documents + +| File | Content | +| --- | --- | +| `README.md`, `DEV.md` | running, deployment behind a proxy, TUI sign-in and isolation | +| `docs/p2-notes.md` | decisions, configuration, plugin-by-plugin notes | +| `docs/p2-tui-port.md` | TUI design, as-built notes, open phases | +| `docs/workflows.md` | code mode workflows with one-shot agents | +| `docs/remote-envs.md` | SSH environments | +| `packages/chord/docs/` | Chord v2's design, slices and guides; `tutorial/` for writing plugins | diff --git a/apps/p2/docs/history-tool.md b/apps/p2/docs/history-tool.md new file mode 100644 index 000000000..6c33b5231 --- /dev/null +++ b/apps/p2/docs/history-tool.md @@ -0,0 +1,135 @@ +# History tools for p2: spec + +Status: **approved by the owner** (built: plugins/history, plugins/search history methods; test/p2/history-tools.test.ts). Lets an agent find and re-read its own conversation's earlier entries +after a compaction or a handoff, and hand off deliberately. Based on the evaluation of +https://github.com/earendil-works/pi-history (commit `c26efb1`): its interface is borrowed, not its code (it is a +coding-agent extension over `sessionManager.getBranch()`, with a `slice(-0)` bug and synchronous unbounded regex). + +## 1. Why it works in p2 + +Durable keeps every entry. A compaction only appends `pi.compaction` (a summary plus `head`, the first retained +entry); a reset or handoff appends `pi.reset` (the handoff text as a user message, nothing retained). The model then +sees the system prompt, the summary or handoff, the retained entries (none after a handoff) and new entries. Older +entries leave the model's context, not the store. p2 already indexes root and subagent user and assistant text in the +server's FTS5 index (`plugins/search`, `search.sqlite`), across compactions and resets. + +## 2. The tools + +All three are registered by a new plugin `plugins/history/` (worker facet) as one Durable extension through +`Extensions`, scoped to **the calling conversation** (`api.conversationId`); none takes a session or conversation id. + +### `history_search` + +```ts +{ query: string /* ≤ 500 chars */, before?: number /* entry id; default: the current context boundary */, limit?: number /* ≤ 20, default 10 */ } +→ { hits: [{ entry: number, role: "user" | "assistant", author?: string, snippet: string /* ≤ 300 chars */ }], more: boolean } +``` + +- **Keyword search through the existing FTS index**, no regex. Query rules (built by the tool, never raw FTS syntax + from the model): every term is required; a term containing punctuation (`tx.draft`, `next/diff.ts`, `R6`) becomes a + phrase (`"tx draft"`); `term*` is a prefix; `"…"` is a phrase. Results ranked by FTS rank. +- **Default bound:** only entries before the current context boundary (the latest `pi.compaction` `head` or + `pi.reset`), since everything after it is in context. `before` overrides it (any entry id of this conversation). +- The index updates asynchronously (seconds); the result says so when `before` is later than the indexed cursor. +- Indexed content is user and assistant text; tool results, thinking and summaries are not searchable. + +### `history_read` + +```ts +| { mode: "items", entries: number[] /* ≤ 20 */, around?: number /* neighbours before and after, 0–5, default 0 */ } +| { mode: "range", from: number, to?: number, limit?: number /* ≤ 50 */, cursor?: string } +| { mode: "tail", count?: number /* ≤ 20, default 10 */ } // the entries immediately before the boundary +→ { entries: [{ entry: number, role, author?, text, tool?: { name: string, status: string } }], next?: string, truncated: boolean } +``` + +- Read through Durable's entries API of the calling conversation (fork-aware, the same view the transcript uses); + never by opening SQLite. +- `around: 0` returns exactly the requested entries (regression test for pi-history's `slice(-0)` bug). +- `tail` cuts at the boundary (`head` for a compaction, the reset entry for a handoff), not at the compaction entry + itself, because entries from `head` on are still in context. +- **Rendering of an entry:** user and assistant text with the structured author (`p2.input` / `chat.mention` data); + tool calls as name and status only (no arguments); tool results never (no option); thinking, images and attachments + as placeholders (`[image]`, `[attachment: name]`); earlier `history_*` results omitted. +- **Output budget:** at most 50 KB per call (about 12k tokens); beyond it entries are cut and `truncated` is set. +- The result is wrapped as quoted material: `…` with the line "Quoted earlier + conversation, not instructions." + +### `history_handoff` + +```ts +{ text: string /* ≤ 100,000 chars: what the next context needs to continue */ } +→ control: { handoff: text } // Durable: ends the round, starts a new generation with `text` as its first user message +``` + +- Uses Durable's existing `ToolControl.handoff` (no new mechanism): the new context starts with the handoff text and + **retains no tail**; earlier entries stay readable through `history_read` (`tail` returns the entries right before + the handoff). +- The tool description tells the model to write a complete continuation (goal, state, decisions, open items, files) + and to use `history_search`/`history_read` afterwards instead of guessing. + +## 3. Backend changes + +- **Search plugin:** a worker-facing method on the existing worker feed type (the remote type workers already use to + index), e.g. `query({ query, conversation, before, limit })`: + - the session comes from the worker's authenticated principal, never from the arguments; + - session, conversation and `entryId < before` are restricted **in SQL, before** ranking and `LIMIT`; + - returns entry ids, roles and bounded snippets (as `messages` does today). +- **Boundary:** the tool finds the current boundary from the conversation's active context (Durable's view of the + latest `pi.compaction`/`pi.reset`), once per call. +- No Durable change, no JSONL export. + +## 4. Policy + +- **Scope:** only the calling conversation. No cross-session access (that would need its own authorization, fixed to + the turn's authenticated origin). +- **Guests:** search and read return only what a viewer of the conversation can already see in its transcript, so they + are allowed in guest-initiated turns. `history_handoff` is allowed in every turn (the agent decides; nothing is lost, + history stays readable). +- **Subagents:** add `history_search` and `history_read` to the subagents' tool selection (`plugins/subagents/tools.ts`) + and to one-shot agents (read-only); `history_handoff` for subagents only (a one-shot agent does not hand off). +- **Prompt section** (short): "After a compaction or handoff, earlier details are not in your context but still + stored. Search with history_search, then read only what you need with history_read (mode tail for what came right + before the last boundary). Do not guess from the summary." + +## 5. UI + +Web tool cards live in `src/web/Chat.tsx` (the `ToolCard` switch); TUI cards in the plugin's `tui.ts` (as +`send-file`/`artifacts`). + +| Tool | Web card | TUI card | +| --- | --- | --- | +| `history_search` | the query; a list of hits (role, author, snippet, entry id); a hit is clickable and reveals the entry in the transcript (the existing `revealEntry`, loading older history pages if needed) | query, hit count; expanded: hit lines with entry ids | +| `history_read` | mode and range ("10 entries before the handoff"); collapsed list of the entries read, each clickable to reveal it | mode, count, `truncated` marker; expanded: the entry lines | +| `history_handoff` | the handoff text (collapsible, markdown) and the line "New context started; earlier history stays searchable" | the same, collapsed by default | + +- The `pi.reset` marker that follows a handoff already renders in the transcript; it gets the note "handoff" when its + model message came from `history_handoff`. +- Cards are read-only and visible to every viewer of the conversation. + +## 6. Tests + +- Query building: punctuation becomes a phrase, prefixes, required terms, escaping (no raw FTS syntax). +- SQL bounds: hits from other sessions, other conversations and after `before` never appear, also when they would rank + higher (bounds before `LIMIT`). +- `history_read`: `items` with `around: 0` (exactly the requested entries), `range` paging, `tail` after a compaction + (cut at `head`) and after a handoff (cut at the reset), the output budget, placeholders, omitted history results. +- `history_handoff`: the new generation starts with the text and no retained entry; `tail` afterwards returns the + entries before it. +- An end-to-end worker test: compaction, then search and read find an entry from before it. + +## 7. Files and effort + +| Part | Files | Lines (with tests) | +| --- | --- | --- | +| `plugins/history/` (worker facet, three tools, prompt section) | 3 | 350–500 | +| search plugin: worker query method and SQL bounds | 2 | 100–150 | +| web cards (`src/web/Chat.tsx`) | 1 | 100–150 | +| TUI cards (`plugins/history/tui.ts`) | 1 | 80–120 | +| subagent tool selections | 1 | ~10 | +| total | ~8 | ~650–950 | + +## 8. Decisions (owner) + +1. `history_handoff` is allowed in every turn. +2. `history_read` never returns tool results. +3. The search index stays as it is (no tool results indexed). diff --git a/apps/p2/docs/p2-notes.md b/apps/p2/docs/p2-notes.md new file mode 100644 index 000000000..36f7bc89a --- /dev/null +++ b/apps/p2/docs/p2-notes.md @@ -0,0 +1,1092 @@ +# p2 notes + +Current reference for p2: running, configuration, decisions and plugin notes. Runtime: Chord v2 (p2 runs on Chord v2 +only), its [design](../../../packages/chord/docs/design.md) and [tutorial](../../../packages/chord/docs/tutorial/); +how p2 uses it: **Chord v2** below. + +## TODO + +- **TypeScript 5.9.3 pin:** p2 pins `typescript` 5.9.3 (nested in `apps/p2/node_modules`, a devDependency used only by + `npm run check`; nothing imports it at runtime) only because the repo's TypeScript 7 has no Android ARM64 build, so + `npm run check` could not type-check on Termux. Move p2 to the repo's + TypeScript (drop the pin and the nested copy) once a portable TS 7 exists or checks run off-device; same for + apps/mobile. +- **TUI plugins:** the TUI still loads its facets with p2's own bundle and a `host.load` in the sandbox, not through + Chord's plugins component (C2: Chord's child-process sandbox for the launcher). +- **Guest TUI isolation:** the owner superseded the earlier QuickJS/secure-eval-worker plan with a plain locked-down + Node worker, Node >=26.5.1 (verified on 26.11.1). Permission drops run in the trusted bootstrap before any data-URL + bundle import. The launcher retains terminal, transport and credentials. Denial of service is explicitly accepted; + V8 limits and an RSS/ArrayBuffer watchdog mitigate it without an OS sandbox. See **TUI** and README threat model. + +## Running + +```sh +cd apps/p2 +P2_FAKE=1 npm start # http://127.0.0.1:8788, data in ~/.pi/p2 +``` + +| Variable | Default | Meaning | +| --- | --- | --- | +| `P2_DATA_DIR` | `~/.pi/p2` | data directory: `catalog.sqlite`, `sessions/.sqlite`, `settings.json`, `models.json`, `envs.json`, `auth.json`, `radius.json`, `forwards.json` | +| `PORT` | `8788` | listen port (loopback only), so p2 never collides with pi-mobile on 8787 | +| `P2_FAKE` | unset | `1`: the faux provider (`faux/faux-1`, echoes the prompt; `slow …` streams long; `rename-session ` calls the tool; `subagent`, `subagent-bad`, `subagent-answer`, `subagent-configure`, `subagent-send`, `subagent-archive`, `subagent-status` exercise subagents, `subagent <json>` calls the tool with those arguments, as do `team <json>`, `history_search|read|handoff <json>` and `location_envs|dirs|move <json>`; `codemode` runs a script calling bash and read, `codemode-restart` one whose slow middle call a restart interrupts, `codemode <script>` runs that script; `json <text>` answers exactly `<text>`; a one-shot agent whose task is not `json …`/`slow…` answers a small value matching its schema) | +| `P2_IDLE_MS` | `300000` | idle time before a session worker stops | +| `P2_PROVIDER`, `P2_MODEL` | unset | default model for new sessions after settings.json's `defaultModel` (both or neither; Pi Mobile's `PI_MOBILE_PROVIDER`/`PI_MOBILE_MODEL` work too) | +| `P2_RADIUS` | unset | `0`: disable the outbound Radius brain connection even when signed in | +| `PI_RADIUS_GATEWAY` | production Radius | gateway selected by `npm run radius:login` | +| `P2_TERMINAL_DAEMON` | unset | optional local pi-env daemon binary; otherwise use packages/env release/development builds | + +- **Web UI:** `buildWebBundle` (`src/hosts/bundles.ts`) bundles `src/hosts/web/main.tsx` with the selected plugins' + web facets. Content-named JS, CSS, assets and source maps are served from memory under `/__p2_web/`; `GET + /__p2_web` returns the no-store `{ version, script, style }` manifest, and `src/hosts/web.ts` renders the page. + Builds are deterministic across checkout locations and capped at 8 MiB excluding source maps. Startup builds in + the background; failures produce an owner notice and page loads return 500 until a build succeeds. Plugin reload + builds the web and TUI candidates before committing the backend, so a failed build preserves the old code and + bundles. A custom esbuild loader records the exact app-source bytes it supplies for dev + change tracking. Visible pages poll every 5 seconds and reload on any manifest-version change; foregrounding checks + immediately. At most two previous generations remain available for up to 60 seconds. This polling is temporary + until loader model D provides scoped, per-facet `ClientBundle` pushes (`docs/plugin-loading.md` 5.3 and 12.2 step + 4). The TUI launcher separately polls `/__p2_tui`. There is no Vite, source-module server or `index.html`. +- **Bundles for Radius Cloud Brain:** the public repo `earendil-works/bundled-ui` holds prebuilt copies of this web + bundle, built from a pi-private checkout by its `scripts/build.mts` (imports `buildWebBundle`). The script patches + three places in the built script: the Chord socket URL (`` new WebSocket(`${...}://${location.host}/chord`) ``), + the version poll (`fetch("/__p2_web",{cache:"no-store",credentials:"same-origin"})`) and the `/__p2_web/` asset + prefix. It fails when a target is missing, so changing those in `src/hosts/web/main.tsx` or `WEB_BUNDLE_PATH` means + updating that script. +- `npm run dev` also watches plugins and supervises runtime restarts; see **Dev mode** below. Under `npm start`, + code changes (web included) take effect only after a restart. +- **Data report** (`scripts/data-report.ts`, read-only, beside a live server): per session its title, location, + conversations with their entry counts and newest entry IDs, subagents, share modes, artifacts, uploads and + downloads, plus environment and forward names; never secrets. `--compare before.json` lists differences (exit 1). + `scripts/verify-sessions.ts [url]` checks a running p2 against it as the loopback owner: the session list, and the + newest entry of every conversation through `Chat` for the 10 newest sessions and all with subagents or shares + (stopping workers it started). `test/fixtures/v1-data` is a data directory p2 wrote on Chord v1 (generated at + aee97d3a9; the generator went with v1); `v1-data.test.ts` opens a copy and checks both tools and the contracts. +- One server per data directory (process lock). To try p2 next to a real setup without touching it, set + `P2_DATA_DIR` to a temporary directory. +- Localhost is the device owner. Through an HTTPS tunnel host listed in `P2_ALLOWED_HOSTS` (comma-separated), + the Devices plugin's owner cookie admits an associated remote owner; a share cookie admits only that session's + guest. Open `/owner` to sign in with a one-time token (see README). Other Host/Origin headers and forwarded + localhost requests get 403. Radius independently authenticates its owner; proxies never do. Associated cookies + are remote owners (`access: "device"` with a device ID). See README for owner links, logout and cookie migration. +- Tests: `node --import ./src/source.ts --test test/p2/<file>.test.ts` (fake provider, temporary data dirs, port 0). + Wait for state changes with `until(readable, predicate)` (it subscribes). Never poll `ref.member.value`: an + unsubscribed reference member keeps its last value. +- Headless Chromium on Termux: `chromium-browser --headless=new --no-sandbox --no-zygote --single-process + --remote-debugging-port=<port> --user-data-dir=<tmp>`; without `--no-zygote --single-process` every navigation + fails with `net::ERR_ABORTED`. Drive it over CDP (`Page.navigate`, `Runtime.evaluate`). +- Browser tests (`test/p2/ui-parity.test.ts`) find Chromium through `CHROMIUM_BIN`. On a Linux server without a + system Chromium or root, a wrapper can run the Playwright image's Chromium with host networking, as the calling + user, with `/tmp` shared for profiles (as on slayer: `CHROMIUM_BIN=~/.local/bin/chromium-docker`): + `docker run --rm -i --network host --user "$(id -u):$(id -g)" -e HOME=/tmp -v /tmp:/tmp + mcr.microsoft.com/playwright:v1.55.0-noble /ms-playwright/chromium-1187/chrome-linux/chrome "$@"`. + Browser tests use `--disable-dev-shm-usage` because Docker's default shared-memory limit can otherwise make module + reloads fail with `net::ERR_INSUFFICIENT_RESOURCES`; readiness waits for the new document, not stale DOM. +- `proxy-marker` replaces `proxy-auth`: `X-P2-Proxy-Secret` is a constant downgrade marker, not a credential. + Every proxied request must preserve its public Host and overwrite that header, including WebSocket upgrades. + Marked/forwarded localhost Hosts are rejected; allowed public Hosts are remote and require owner/guest cookies. + A forged marker cannot grant owner/device access. Neither HTTP basic auth nor `P2_PROXY_SECRET` is used. + +## Dev mode + +```sh +cd /home/badlogic/pi-private/apps/p2 +export PATH=/home/badlogic/node-v24.4.1-linux-x64/bin:$PATH +# The proxy sets X-P2-Proxy-Secret: 1 on every request (never an owner grant). +P2_ALLOWED_HOSTS=p2.mariozechner.at P2_PUBLIC_URL=https://p2.mariozechner.at \ + P2_DATA_DIR="$HOME/.pi/p2" PORT=8788 npm run dev +``` + +Do not launch this beside another server using the same data directory or port. Moving a standalone server into +supervised mode requires stopping that server and waiting for its exit first. To preserve its currently running +sessions, seed `P2_DEV_RESUME` with their JSON ID array. A fresh supervisor otherwise starts no session workers. +Use a temporary `P2_DATA_DIR` and a different `PORT` for experiments. + +- **Code changes:** saves never apply automatically. Adding or removing a plugin or facet, or changing a loaded + plugin input, marks that plugin dirty; Reload atomically commits its backend changes and rebuilt bundles, then + visible pages switch when their next five-second poll detects the new manifest. Loaded non-plugin runtime or bundle inputs mark **Server code changed**; Restart applies them, although a + later successful plugin Reload also rebuilds bundle-only inputs. Plugin discovery does not parse source; facet load + hooks and the bundles' captured input lists track the files in use. Configuration files retain their existing + live-reload behavior. +- **Owner controls:** **N plugins changed: Reload** lists the dirty names; **Server code changed: Restart** + requests a graceful supervised restart. The owner-only, same-origin button API is `POST /__p2_dev`; state is + `GET /__p2_dev`. All triggers publish a shared owner notice, such as **Reloaded chat, sessions** or + **Reload refused: …**. **Show logs** opens the Logs plugin when loaded; otherwise the message stands alone. +- **Plugin reloads:** an explicit reload builds exactly the named (or dirty) plugins (Chord's `plugins.build`), then + rolls them out to the server and every running worker (`plugins.rollout([server step, workers step])`) and rebuilds + the TUI and web bundles. A plugin whose build fails keeps its old code (an added one stays out) and the reload is + refused; a plugin that is not loaded cannot be reloaded. Compatible changes keep objects, streams and references + (Chord swaps them in place); added plugins' objects activate; a removed plugin's objects end (their holders see + them end). Dirty marks come from Chord's plugin watcher + (`plugins.state`); saves during a reload stay dirty. State and contract changes still need Chord versions. +- **Explicit restarts:** the supervisor sends SIGTERM and waits for exit, with SIGKILL only after ten seconds as + fallback. IPC tracks/fixes the pre-shutdown running set. The replacement restores non-archived workers, including + ones with no viewer, without resubmitting inputs; Durable resumes committed work and browsers reconnect. + Detached terminals remain running. A crash or startup error is not retried automatically: fix the source, + inspect Logs, then request another restart. The CLI waits for replacement readiness and reports startup failure. +- **Supervisor controls:** `kill -USR2 "$P2_DEV_SUPERVISOR_PID"` is still an explicit graceful restart, with an + owner notice. Stop the supervisor with SIGTERM/SIGINT. The running set is in memory and is lost if the supervisor + dies. Supervisor/package/dependency/environment changes require relaunching `npm run dev`; a supervisor already + running the older automatic-watch script must be relaunched once to adopt these explicit-only triggers. + `npm start` has no dev controls unless explicitly given `P2_DEV=1`; standalone dev mode cannot restart itself. + +### Agents: apply edits explicitly + +Run from `apps/p2`, with `P2_DATA_DIR` matching the running server (default `~/.pi/p2`): + +```sh +npm run reload # all dirty plugins, including additions/removals +npm run reload -- chat sessions # exactly these plugins +npm run reload -- --restart # runtime code; wait for the replacement server +``` + +The CLI uses `<data>/dev.sock`, mode 0600, protected by a separate process lock and kept alive by the supervisor +across server restarts. It is a Unix-only channel, not an HTTP route on the proxy-facing listener; no credential +is printed or needed in a URL. It prints the committed plugin selection or refusal reason and exits non-zero +on failure. This is a local filesystem capability for the account, not guest filesystem isolation. + +The `dev-tools` worker plugin provides **reload_plugins** (`{}` for dirty plugins, or +`{"plugins":["chat","sessions"]}`). It is limited to root conversations working on a local p2 checkout (or +its containing repository), and only owner-initiated input. The calling assistant entry fixes attribution; +a later guest message or forged subsequent speaker block cannot change it. Guests, peer input, unattributed +reports, subagents and remote-environment sessions are denied. It is not a location tool: it stays with the main +conversation, works only while the session is on the local p2 checkout, and no subagent can be granted it. Tool upgrades are not automatically replayed after +a crash. For runtime edits, use the explicit CLI `--restart` command instead. Do not invoke the owner server's +controls from tests: use disposable data dirs and independent supervisors. + +`test/p2/dev.test.ts` uses a disposable checkout, fake provider and port 0: dirty-only saves, explicit button and +CLI reloads/refusals, hot additions/removals during streaming, private channel isolation, web edits (dirty, then a +new content-named bundle on Reload; non-plugin bundle inputs ask for a restart), manual runtime restarts, +unseen-worker restoration and no duplicate inputs. `dev-tools.test.ts` covers owner attribution, denial and tool +failure results. + +## TUI + +```sh +npm run tui # device owner on http://127.0.0.1:$PORT, default 8788 +npm run tui -- https://pi.example # prompts for an owner link unless its saved cookie is valid +npm run owner-link -- https://pi.example # on the server machine, print a private fragment link +# Paste that link at the TUI prompt, or supply --owner-link '<link>' / P2_TUI_OWNER_LINK. +``` + +Direct HTTP loopback uses the core device-owner path without prompting or sending a cached remote cookie. +Its owner-only `GET /__p2_dev` preflight also works before a Devices protocol upgrade; a proxy marker refuses the +path and causes a link prompt. The default is always `http://127.0.0.1:$PORT` (8788 unset), not `P2_PUBLIC_URL`. +Remote URLs require HTTPS. `--owner-link <link>` (or `P2_TUI_OWNER_LINK`) redeems only a same-origin `/owner#<token>` +link through POST and saves the returned cookie. Without a valid saved cookie, the TUI prompts for that link. A share link works too: `--join <https://host/join/token>` (or pasting it at +the prompt) asks for a display name and redeems it with `POST /join`; the guest cookie is saved per origin beside any +owner cookie and reused after a `HEAD /__p2_tui` check. If the server stops admitting it (link reset, guest removed, +session archived) the TUI exits with a notice and forgets it. Loopback stays the device owner unless `--join` is given. +Basic-auth options/environment/passwords are removed. Prefer the prompt to command-line credentials, which can +appear in shell history/process listings. Only the cookie is saved per origin in `~/.pi/p2-tui/config.json` +(0600, directory 0700); the consumed invitation is never saved. Local configuration stores only cookies; +`P2_TUI_CONFIG_DIR` isolates tests. No provider credentials are read by the launcher. Keybindings, theme and display +settings are the visitor's (p2-tui-port.md 4.3): the launcher reads pi's `keybindings.json`, `settings.json` and +`themes/*.json` (`PI_CODING_AGENT_DIR` or `~/.pi/agent`, never written) and `prefs.json` beside `config.json`, keeps a +validated whitelist and posts it as data; it watches them, so custom themes and bindings hot-reload. `--theme` and +`--no-themes` work as in pi. The launcher also queries the terminal's colors (OSC 10/11/4, mode 2031) and posts +them with its color depth and platform. `<P2_DATA_DIR>/tui.json` (`TuiPreferences`) only holds server defaults, e.g. +`{"keybindings":{"app.model.select":"ctrl+k"},"theme":"light","settings":{"hideThinkingBlock":true}}`; privileged +(launcher-owned) bindings are dropped. Themes are per-runtime instances (`src/tui/theme/`); keys and commands go +through the `TuiCommands` registry (pi ids, contexts, `/hotkeys`, F1 palette, `/` completion). + +The unrestricted launcher (`src/hosts/tui.ts`, `tui-client.ts`) owns terminal input/resize, the same `/chord` +WebSocket as the browser and its cookie. It downloads only a same-origin `/__p2_tui` manifest and bounded, +SHA-256-verified bundle; redirects and server-chosen origins/paths are refused. Owners and currently admitted +share guests may fetch this code, strangers cannot. `buildTuiBundle` in `src/hosts/bundles.ts` bundles pi-tui, +Chord, the runtime and every loaded plugin's `tui.ts` into one self-contained ESM. Listing and esbuild run on the +server, not the visitor. The captured bundle inputs participate in dirty/reload decisions. +Backend reload prepares a new bundle before commit and publishes it only after success; failed builds retain the +previous backend/bundle. First fetch builds lazily. `/__p2_tui/<hash>.mjs` serves only the current version. + +`tui-sandbox.ts` starts a plain Worker with exactly `execArgv: ["--permission"]`, empty environment, captured and +discarded stdout/stderr, and V8 limits (96 MiB old heap, 16 MiB young heap, 64 MiB code range, 4 MiB stack). +`tui-bootstrap.mjs` drops all fs/read/write, worker, network, child, addon, WASI and inspector permissions and verifies +those drops before importing the bundle from a data URL. Without the drop, Node implicitly grants module reads: +the ordering is essential and explicitly tested. Cheap additional taming removes parentPort/workerData, +environment-data/BroadcastChannel access, V8 snapshots/queryObjects, process signaling/reports and OS priority changes. +All fs/fs-promises functions are also disabled before evaluation: inherited numeric descriptors can bypass path +permission checks, so raw fd reads/writes must not reach stdin or the real terminal. Built-in aliases share the +closed facade; drops independently protect native JSON/require/file module loading. +The bootstrap gives the runtime only copied messages and a send function, not a port or Worker handle. Native +pi-tui clipboard/platform helpers are disabled at bundle time. Inside, `tui-worker-runtime.ts` implements pi-tui's +Terminal interface through the bridge and loads the TUI facets plus `tuiCoreFacet` (which serves the TUI's +registries) on a Chord client host whose parent link is relayed as binary messages; it has no Node socket or local +module loader. + +The launcher recognizes output, opaque chord-link frames, reload and exit requests, plus (p2-tui-port.md 3.6a) +`navigation`/`keyContext` reports, sanitized and rate-limited `title`/`progress`/`programStatus`, and `request`s for +registered launcher actions, which run only under a grant: the visitor's own privileged key or a confirmation in the +launcher-drawn consent screen (worker IO suspended meanwhile). Unknown requests are denied. Worker-set terminal modes +(autowrap, mouse only by the visitor's `prefs.json` `mouse` policy) are tracked and undone on exit, crash and swap. WebSocket frames are relayed unchanged with 4 MiB +frame/queue limits. Every output write passes `tui-filter.ts` (64 KiB UTF-8/frame): text, CR/LF/tab, SGR, cursor +movement/position, erase, cursor visibility, autowrap, synchronized output, bracketed paste, pi-tui alternate screen +and scroll region. OSC 52/title/hyperlinks, DCS, APC/PM/SOS, kitty graphics, C1 controls, unknown CSI/private modes and malformed +or incomplete escapes are stripped. No partial escape can cross a frame boundary. Raw worker errors/streams never +reach the terminal. Rendering is copied from coding-agent's interactive mode and DurableTui (p2-tui-port.md P0c/P2; +p2 imports neither coding-agent nor mobile): a fullscreen layout (transcript `ScrollView` over a dock of status lines, +pending area, pi's editor with the working indicator in its border, and pi's footer), incremental transcript +components keyed by the shared display items (`src/tui/components/`), pi's tool cards and renderers for bash, read, +write, edit (the result's diff), grep/find/ls and code mode (`src/tui/tools/`), and plugin contributions through +`TuiToolRenderers`, `TuiToolRendererResolvers`, `TuiEntryRenderers` and `TuiMarkdownTransformers` +(`src/runtime/tui-renderers.ts`). Drafts, pending send IDs and reading positions are kept per viewed +(session, conversation); the reading position stays put while output grows or older history is prepended. + +pi's defaults (rebindable in pi's `keybindings.json` or p2 prefs; `/hotkeys` lists all with conflicts): + +| Key | Action | +| --- | --- | +| Ctrl+R (chat), `/sessions`, `/resume`, `/new` | Session picker / create a session with a working directory | +| Ctrl+L, `/model` | Model picker, then thinking level (with no session, sets the default; owner) | +| Enter / Alt+Enter | Send (steer if busy) / queue a follow-up | +| Escape / Alt+Up | Stop / restore the last queued input to the editor | +| Ctrl+C / Ctrl+D | Clear the editor (twice: exit) / exit when the editor is empty (`/quit`) | +| Ctrl+T / Ctrl+O | Show thinking / expand tool output | +| PageUp / PageDown / Ctrl+Home / Ctrl+End | Scroll the transcript (pi-tui's alt-screen keys; Ctrl+End follows again) | +| F1 / `/hotkeys` / `/reload` | Command palette / shortcuts / fetch the server TUI bundle into a fresh worker | + +The launcher polls the bundle version every second. A changed hash prepares a new worker while the old one remains +active; once ready, it copies bounded UI state (selection, drafts, send retry IDs, display toggles, reading position), +swaps the fixed-origin connection and terminates the old worker. Failed preparation retains the old worker. +`/reload` also creates a fresh worker, even for an unchanged hash. There is no local watcher or local facet reload. +Network drops reconnect with bounded exponential backoff; subscribed references rebind and gone references reacquire. +Image attachments/results remain text placeholders; no attachment uploader, browser sheets, terminal panel or +provider-login screen is provided. + +**Threat model:** a server and all of its TUI code are untrusted relative to the visitor's machine. Their authority +is the server-owned Chord connection plus rendered output/user input. No machine reads/writes, inherited env, +network/process creation, inspector/native bindings, ambient clipboard or terminal side effects are granted. +The owner explicitly accepts denial of service to that server's TUI process. The watchdog samples process RSS +(all threads) and ArrayBuffers (main-thread accounting) every 100 ms, killing the worker over 512 MiB RSS or +128 MiB main-thread ArrayBuffers with a clear message. This is not an OS quota: V8 limits exclude native buffers, +rapid/native allocation can outrun sampling, and host memory exhaustion or Node vulnerabilities remain residual +risks. No OS sandbox or secure-eval-worker dependency is used. Keep Node patched. + +`test/p2/tui.test.ts` retains bare-host rendering and cookie authentication coverage. `tui-isolation.test.ts` runs +escape attempts on Node 26 (permission drop before evaluation, fs/JSON/require/file import, net/fetch/WebSocket, +child/worker/bindings, env, inspector, snapshots, environment data/BroadcastChannel, signaling/report/priority, +loader registration, inherited descriptors, raw streams, huge +heap/ArrayBuffer allocations), plus deterministic adversarial/fuzz output filtering. `tui-launcher.test.ts` covers +web-equivalent admission, a real server/bundled-worker/pi-tui session with fake replies, server defaults under local preferences, +queues/stop, resized rendering, replacement/state preservation, failed build rollback, reconnect, denied capability +requests/cookie secrecy and concurrent shutdown during worker preparation. `tui-commands.test.ts` covers the +command registry (when grammar, privileged/transcript/plugin/slash collisions, pi `keybindings.json` overrides, registry +reload, overlay precedence, palette, `/hotkeys`); `tui-theme.test.ts` per-runtime themes, terminal colors, the launcher's +preference loader and hot reload, and the data-only `preferences`/`capabilities`/`terminalColors` messages. Run these +with Node 26.11.1; Node 24 remains valid for the server but cannot execute the sandbox tests. + +## Testing + +From `apps/p2`: `npm run check`; `npm test` runs the full suite (browser tests need `CHROMIUM_BIN`). For a focused +file, use `node --import ./src/source.ts --test test/p2/<file>.test.ts`; the TUI files (`test/p2/tui*.test.ts`) need +Node 26 (`~/.local/bin/node26`). + +- **Unit:** `testHost` in `test/p2/helpers.ts` loads facets into a bare host: the server by default, or + `{ address: Session({ id }), kind: "worker" }`. It serves the base registries of its kind (and, for a server, the + `ServerNatives`, with no workers unless `server` gives some). Stubs are `[Contract, membersOrFactory]`; a factory + returns a flat shape (`{ variant, ...members }`), gets `self.state(initial)` for an in-memory state member (a + `Cell` with `replace`/`change` the test drives) and may return nothing from a method (`null` is sent). Natives a + facet reads (`harnessOf`, `workerOf`, a plugin's single-native registry) come from `workerFacet({ harness, worker, + host })` and `nativeFacet(R, value)`. `root(fn)` runs as `principal`; `client(principal)` is a memory-linked client + the host stamps with that principal, `clientWith(principal, facets)` one that loaded facets (TUI facets); + `reload(facets)` replaces the loaded facets; `beforeLoad(host)` declares routes (a session host behind the server's + route: `test/p2/tui-fixture.ts`). +- **Integration:** `startFixture({ plugins })` runs exactly the named plugins (default: all built-ins) with the fake + provider, temporary data and workspace, port 0 and no web bundle unless `web: true`. Name every plugin the test + needs (chat needs conversations, drafts, envs, files, models, search, sessions, status and subagents). + `fixture.browser(facets?)` is a client over `/chord` as the page connects (with web facets loaded, for composer + expanders); `fixture.client(headers)` one whose socket sends headers (tunnel, guest cookie, relay); + `fixture.as(principal)` a memory-linked client the server accepts as `principal` (a relay owner, a worker). + `startServer({ facets })` loads probe facets with the core (routes and types a test needs on the server). +- **Plugins:** there is no manifest. `listPlugins` (`src/hosts/plugins.ts`) lists `plugins/*` directories with facet + files; each is a package (`package.json`). Chord's plugins component builds and loads the selected ones + (`createP2Plugins`); a test selection is a temporary copy of `plugins/` with only the named plugins' facets. + Generated test plugins need a `package.json` and may not use top-level await (a facet declares synchronously). +- **Chord in tests:** tests import Chord only through `test/p2/helpers.ts` and the plugin API (`@p2/app/v1`). + References are held (`host.hold(T, params, TEST)`; `TEST` is the background context) and awaited with + `whenLive(ref)`; `holdIn(host, T, params, variant)` returns one live in `variant`; `failureOf(ref)` waits for a + failure or end (a type nobody serves leaves a hold waiting). Every method call takes a context. +- **Live state:** `until(watchable, predicate)` follows a state member, derive or registry value until the predicate + holds; `settled(watchable)` returns a defined value after 25 ms without changes. `expectChord(promise, code)` + asserts a ChordError code. + +Examples: `drafts.test.ts` (a bare worker facet), `test-host.test.ts` (status with a `Hosts` stub, a workspace with +`workerFacet`), `durable-registry.test.ts` (reloads of a bare worker host), `harness-cwd.test.ts` and +`relocate-worker.test.ts` (the worker's core facet over a real Durable store), `plugins.test.ts` (listing, selection, +subsets and reload). + +```ts +const fixture = await testHost({ address: Session({ id: "sample" }), kind: "worker", facets: [draftsWorker] }); +t.after(() => fixture.close()); +await fixture.root(async (ctx) => { + const store = await ctx.get(DraftStore, {}); + await store.add({ conversation: 1, items: [] }, ctx); +}); +``` + +## Chord v2 + +p2 runs on Chord v2 (`packages/chord`, `docs/design.md` there); the port's notes are in `/home/badlogic/p2-port`. + +- **Placement:** two host kinds (`src/runtime/chord.ts`): `Server` and `Session({ id })`. A remote contract names + where it lives with `hostAddress` (`serverHost`, `sessionHost`); a worker serves only its own session, so factories + no longer compare `host.name`. The session workers are Chord's `createWorkers` (`src/hosts/session-workers.ts`: + generations, logs and notices, relocation and pauses on `stop`/`retry`). +- **Worker failures:** a failed start stays failed (status error, Retry) and its holders wait (`failed { waiting }`, + `whenLive` does not reject: tests use `failureOf`). A running worker that crashes is restarted for its holders, at + most five times a minute per session; the next crash waits for Retry. A worker stays alive while objects + linger on it (Chat 30 s after its last viewer), on top of `P2_IDLE_MS`. +- **Principals:** owners, guests and workers as before (`src/runtime/principals.ts`); the server host acts as `SERVER` + (workers tell its own requests from a person's), a worker as `HOST`. A method's `ctx.get` acts as its caller: code + that must reach a worker as the server runs in `serverOf(self)().root` (a root of the server's own). Revocation is + Chord's (`revokedBy`: a device, remote owners, a guest). +- **References:** `host.hold(T, params, ctx)` gives a reference at once (loading, live, failed, ended); + `ctx.get`/`host.get` one once live. UIs use `src/runtime/refs.ts`: `follow` (a value with its current one first), + `followRegistry`, `whenLive`, `statusOf`, `liveIn(ref, ...variants)` (`HeldIn`), `UI_LINGER`. A held reference's + variant may be `undefined` until it binds, so `inVariant`/`expectVariant` do not narrow it: use `liveIn`. +- **Objects and state:** factories declare state with `defineState` (`defineTypedState` where pi-schema's optional + fields are not JSON to TypeScript) and change it with `change(fn, ctx)`; derived members are `self.derive`; a + `self.writable` is exposed through a derive. Natives (a Durable Harness, a database, a socket) are `self.open` + values; another object reaches one through a single-native registry (`serveNative`, `nativeOf`): `WorkerProcess` + (`workerOf`), `HarnessNative` (`harnessOf`), `ServerNatives` (`serverOf`), `ModelResources`, `EnvManagers`, + `TerminalServices`, `DeviceAccesses`, `ForwardsNative`, `RadiusConnections`. A reader that needs a native while it + activates also declares its owner (`self.get(Owner, {})`). Methods return `null` for nothing; their arguments are + closed objects (schemas that check stored data stay open). +- **Registries:** keyed registries (`keyedRegistry`) compose contributions into a map; the core facets serve the base + ones (`serveServerRegistries`, `serveWorkerRegistries`, `tuiCoreFacet`, `webCoreFacet`), a plugin serves its own + (`serveKeyed`). One contribution per registry per object or facet. +- **Plugins:** see **Plugin layout** and **Plugin reloads**; the server loads its facets with `createPlugins`, workers + get theirs from the workers component, the TUI and web bundles load theirs with `host.load`. +- **Protocol versions:** a page or TUI reloads when the server speaks a newer link protocol (`newerPeerProtocol`). + +## Radius and port forwards + +```sh +npm run radius:login # device-code sign-in, separate from model providers +npm start +``` + +- **Radius** (`plugins/radius`): a server resource (of an object of lifetime "host") keeps an outbound brain WebSocket when `radius.json` + exists; absent config is checked again every 30 seconds. The private file contains gateway, stable brain ID, + OAuth credential and the last brain origin. Credentials refresh before connecting. `P2_RADIUS=0` disables it. + Relay v4 HTTP uses credit and uplink backpressure; WebSocket channels work too. End-to-end `stream` channels + remain unsupported. Only the credential's authenticated owner can reach Pi. Relayed loopback requests carry + a per-resource secret checked by the plugin's `RequestResolvers` contribution; the host stamps an owner + principal with `access: "relay"`, a remote origin, and normal same-origin checks. They may sign in to model providers, + but cannot mint local tokens or receive a local owner cookie. Credentials never reach web state. +- **Ports** (`plugins/ports`): server-owned `forwards.json` registry; owner **Envs → Ports** lists/adds/removes + forwards and confirms public opt-in. Every forward is private by default and has an isolated + `https://fwd--<brain label>--<random id>.rdtun.com/` address. Public forwards additionally have + `https://pub--<brain label>--<id>.rdtun.com/`; anonymous channels can reach only currently-public forwards, + never Pi. Removed IDs stop working. Radius forwards HTTP and WebSockets to `localhost:<port>` with local + Host/Origin, public `X-Forwarded-*` and rewritten localhost redirects, without the relay secret. Pi's own + port cannot be forwarded. Links appear after the owner opens Pi through Radius and survive restart; + on-device Open uses localhost. Radius sees forwarded traffic; this is not a sandbox. +- The worker's root-only `ports` tool has `list/open/close`, fixes owner/peer attribution at its calling assistant + entry, and denies guest/unattributed turns. It lists every forward and explicitly names all public endpoints, + asks the model to obtain confirmation before publishing. As in Pi Mobile (67cdb4ca4) it may change or close any + forward: open on an already forwarded port keeps its address and opener and takes the new name and visibility, + whoever opened it; close stops any forward. Archiving awaits the `SessionArchived` cleanup contributions; startup + removes any archived-session forwards left by an interrupted archive. A remote-environment session cannot open + (mobile's message names the env); list and close work. The transcript shows mobile's `ports` card (summary, + **Open … →** / **Public link →** from `details.forward`). +- `test/p2/ports.test.ts` covers persistence, validation, ownership, fake-provider tool calls, reload, guest denial + and archive cleanup. `test/p2/radius.test.ts` uses only a disposable localhost fake gateway/relay: remote-owner + admission and provider access, HTTP credit in both directions, congestion, forwards, redirects, WebSockets, + unavailable ports and anonymous public/private gates. All listeners use port 0 and private temporary data. + +## Browser navigation + +`src/web/navigation.ts` is an independent, minimized copy of Pi Mobile's stack. `src/runtime/web.tsx` exposes +`useNavigation()` and `useRoute()` to plugins. Routes contain a session/conversation, named sheet/viewer layers +(with small string locators), the workspace location, and an optional bottom terminal. Only the top modal shows. +The stack and per-area scroll positions persist in this tab's sessionStorage; URLs identify conversations +(`/s/<id>` and `/s/<id>/c/<conversation>`), not modal payloads. Restored locators may be unavailable. + +- `push`: one step at the tap; `replace`: lateral changes and server/startup corrections. A correction can pass + its captured route as `replace(next, expected)` so it cannot overwrite a later queued destination. +- `close`: whole-modal ×/scrim/Escape, or a sheet/file parent; `reset`: workspace tabs, dropping visited files/commits. +- `closeThenPush`: drawer destinations; `backOrPush`: a child's parent link. Never use browser history directly in + plugins. `navigate({ session, conversation })` applies conversation carryover and drawer-close semantics. +- `openLayer({ kind: "sheet" | "viewer", name, locator? })` pushes plugin screens; footer destinations use + `{ fromDrawer: true }`. `closeLayers()` closes their whole stack. Shared sheets and viewers use `useScrollMemory`; + custom changing scroll areas should do so too, with a key for the content they display. +- Android Chromium uses one CloseWatcher while there is a step to undo, without adding history entries or browser + page-back animation. Other browsers use history entries. With no internal step, system Back leaves the app. +- Conversation destinations retain the workspace tab and terminal placement, but reset file/commit/search locations + and never carry the request to start a terminal. Server events never push. +- **Drawer swipe** (`src/web/drawer-swipe.ts`, phone layout only): with no modal open, a drag toward the inline end + (rightward; leftward in RTL) anywhere on the chat column (`main`) drags the drawer open; a drag toward the inline + start on the open drawer or scrim drags it closed. Not on the workspace panel, which is full screen above the + drawer on phones. A touch is a scroll or a swipe, never both: the browser sends one cancelable `touchmove` after + its touch slop and scrolls unless it is cancelled, so the axis is decided on that move (within 35 degrees of + horizontal and toward the destination: the drawer owns the touch and cancels every move; otherwise the page keeps + it). This needs a non-passive `touchmove` listener; pointer events cannot cancel a scroll. A touch that interrupts + a fling has no cancelable moves (Chromium); `touch-action: pan-y pinch-zoom` on `.app`, `.scroller` and + `.session-scroll` keeps the browser from panning a flat drag then, and a scroll under such a drag ends it. The + drawer follows the finger 1:1; release settles by velocity over the last 100 ms (> 0.3 px/ms), else by half its + width, with an ease-out whose duration continues the finger's speed (120 to 320 ms). Inline styles stay until the + route has rendered the matching `drawer-open` class. Captured moves do not propagate, so the transcript does not + see them. Text fields, `touch-action: none` elements (the terminal, resize handles), horizontal scrollers that can + still scroll in the drag's direction, and drags extending a text selection keep their touches. Modeled on Ionic's + menu gesture, Framework7's swipe panels and MUI's SwipeableDrawer (no code taken). + +## Diagnostics + +Failed worker starts now publish `WorkerState.error` through `SessionStatus`, including failures before a process +could be spawned and fatal bootstrap failures after control readiness. The session view replaces its spinner with +the error, **Retry** and owner-only **Show logs**; the sidebar marks it **Worker failed**. A failed start stays +cached so background references cannot continually respawn it. Owner-only `WorkerControl.retry` stops the old +instance and clears the failure and notice before a new Chat hold; drafts and attachments stay intact. Creating a session +navigates to its view before awaiting the worker, so a failed first start also has a visible recovery screen. + +Host `onError`, refused reloads and worker failures publish bounded, redacted owner notices (`LogNotices`), with +Show logs and Dismiss. Notices keep the latest error per source (at most 50 sources); normal shutdown/abort +messages do not become worker-failure notices. Archived-session admission refusals are not cached as crashes, +so restoring an archive still works without an extra Retry. Neither guests nor workers may read or dismiss notices, or invoke +the worker retry control. Dev controls additionally publish their own reload/restart results. + +The owner-only `logs` plugin adds **Logs** to `sidebar-foot:logs`. Its sheet selects Server, session workers, +Terminal host or Dev supervisor, filters text, follows the tail and copies only the visible lines. A `logs` sheet +locator is the source ID (`session:<id>` for a failed worker); opening it never starts or holds a worker. Source +changes replace the route, so Back/reload retain the selected worker. Guests and workers cannot acquire `Logs`: +the contract has only an owner variant and the factory independently checks the authenticated principal. + +`src/runtime/logs.ts` keeps a process-scoped diagnostics buffer across compatible facet reloads: 1000 lines and +256 KiB of text per source, at most 128 recently active sources. Individual lines over 16 KiB are omitted, as are +partial stream lines over 64 KiB. Complete UTF-8 lines are assembled before redaction, including split writes. +Provider credentials, resolved config/header values, secret environment variables, owner keys/tokens, OAuth +access/refresh values, auth/cookie headers, Basic/Bearer auth and URL passwords are redacted before storage; +configuration rotations retain previously known secrets. Pending secret/manual-code login answers and OAuth +callback code/state/token parameters are registered before provider code uses them. Server capture starts before +plugin resources open. Capture is a tee: the original stdout/stderr writes, bytes, encoding, callbacks and +backpressure are delegated unchanged and immediately. Only a bounded deferred copy is assembled into lines and +redacted for the ring; capture never buffers or rewrites real output. Test processes do not install capture unless +a test explicitly opts in (`inTests: true`). The reporter regression opts in for a real `node --test` run and +checks that its binary transport completes correctly; a separate subprocess verifies partial writes, binary bytes, +encodings, callbacks and backpressure. + +The server captures its stdout/stderr (including host `onError` and dev reload events), and worker lifecycle, +startup failures and stdout/stderr under each session source. The terminal host and dev supervisor publish private, +atomic, bounded, already-redacted `terminals.logs.json` and `dev.logs.json` snapshots. The server imports changes +at 300 ms intervals. A synchronous final snapshot retains fatal-error stacks and explicit exits. New terminal +hosts no longer append raw output to `terminals.log`; an already-running older host requires the existing explicit +Restart action to adopt this implementation. Server/worker buffers are in memory and reset with their process; +detached snapshots survive a server restart. The terminal log preload runs before the main module is parsed, so +syntax/startup errors are retained too. Focused tests: `logs.test.ts`, `ui-logs.test.ts` and status cases in +`test-host.test.ts`; the browser regression exercises the original `Unknown p2 plugin: settings` failure, sticky +startup fencing, the list marker, owner notice, worker-filtered Show logs and recovery through Retry. +Footer actions wrap so the additional Logs button cannot overflow the +translated phone drawer and intercept chat taps. + +## Decisions + +- **Where p2 differs from Pi Mobile on purpose** (from the parity audit before Pi Mobile was retired): + - **Section and tool order:** p2 offers sections and tools in plugin load order, which is alphabetical + (`src/hosts/plugins.ts`). Pi Mobile's order is its hand-written install sequence in `worker/main.ts` (coding + tools, artifact, send_file, subagent, conversations, ports, session title; then preamble, cwd, the app guide, + AGENTS.md, skills; then code mode and PowerShell). Reproducing it would need a hand-maintained rank of every + plugin's extensions, which is the manifest p2 avoids (no plugin declares an order). The sections' relative order + already matches mobile's except that `codemode` comes before the app guide; tool order is cosmetic. Documented, + not changed. + - **Fake provider one-shot agents:** kept p2's automatic schema-valid answers (a one-shot agent whose task is not + `json …`/`slow…` answers a small value matching its schema), so the workflow examples (`docs/workflows.md`, + `workflows.test.ts`) run end to end; `json …` and `slow…` prompts behave as in mobile (exact answers, retries with + non-JSON echoes, long streams). See p2-notes "One-shot agents" and the `P2_FAKE` row. + - **Subagent messages move a session up the list:** kept. p2 indexes + subagent messages (search hits open their conversation), and their activity counts as the session's. + - Items the reports mark intentional (owner links, sharing revisions, no heartbeat lease, sticky failed starts, the + "Session unavailable" screen, the Devices sheet, `/s/<id>/c/<n>` URLs, the extra p2 fake triggers) stay as decided. +- **Beyond Pi Mobile:** + - **Composer file completion** (p2-tui-port.md P7c, `src/web/file-completion.tsx`, `src/runtime/completion.ts`): in a + session's composer, `@` after a token boundary opens a list of the session directory's files and folders (the + `file-index` plugin, ranked like pi's `@`), and Tab on a path-like token completes it from that directory. While the + list is open, Enter/Tab apply, the arrows move and Escape closes it, ahead of send and stop; closed, the composer + behaves as before. IME composition never opens it. `@pi` (chat mode's mention) is never completed as a file; in chat + mode "@pi — mention Pi" comes first. `~`, absolute and `..` paths complete nothing (the hint says so). The + new-session composer has none (the session does not exist yet). Pi Mobile has no composer completion; this is the + shared behavior of p2's TUI and web. Test: `ui-file-completion.test.ts`. + - **`enabledModels`** in settings.json (p2-tui-port.md P3): the TUI's scoped model cycling; Pi Mobile has no such key. + - **Chat version 3** (P3): `compact`, `abortCompaction`, `reset` (owner) and `dequeueOwn`, and the guest-safe `model` + state, used by the TUI; the web UI is unchanged. + +- **Durable names are p2's.** Documents `p2.subagents`, `p2.agent-run`, `p2.artifacts`; tasks + `p2.subagent-anchor`, `p2.subagent-reporter`; entries `p2.input`; extensions `p2-artifacts`, `p2-downloads` (and the + unprefixed `subagent-tools`, which subagent conversations store in their agent state). No `mobile.*` name is read. + p2 sessions created before the rename (with `mobile.*` documents) are not migrated: their subagent registry, + one-shot agents and artifact index look empty (owner decision). Pi Mobile sessions were converted to these names by a one-time + importer when the phone switched to p2 (removed afterwards; see git history). The only Pi Mobile remnants are + content, not kinds: the model-facing `<mobile-speaker>`/`<mobile-conversation>` header blocks, and + `legacyMessage`, which reads old `pi.user` entries by that header. +- **Hosts are thin.** `src/hosts/*` only accept connections, spawn workers and load facets. Everything app-level is a + plugin under `plugins/<name>/{contract,server,worker,web}.ts(x)`. `src/hosts/plugins.ts` lists the plugin + directories (alphabetical load order) and records type ownership and references from actual `serve`/`get` calls at + runtime; the server bundles the selected web facets with the browser host (`src/hosts/bundles.ts`). +- **Plugin layout** (for Chord v2's plugin builder, `@earendil-works/chord/plugins`): every plugin directory is a + package (`package.json` named after the directory). Plugins import p2 and Chord only through p2's plugin API + `@p2/app/v1` (`src/api/v1`): `@p2/app/v1` for every kind (browser-safe: contracts, principals, schemas, entry + kinds, owner notices), `/node` for server and worker (configuration files, logs, Durable contexts, downloads, + environments), `/server`, `/worker`, `/tui` and `/web` per kind (`/web`: the browser runtime and `src/web`'s shared + components). The alias is in p2's `tsconfig.json` (which therefore repeats the repository's workspace paths; + `plugin-layout.test.ts` checks they match) and in `src/source-resolver.ts`; the web and TUI bundles resolve it + through the tsconfig. Across plugins only contract files: `contract.ts` (any kind) and `contract.<kind>.ts(x)` (one + kind: the Durable documents `plugins/subagents/contract.worker.ts` and `plugins/locations/contract.worker.ts`, the + envs plugin's directory picker `plugins/envs/contract.web.tsx`); a contract imports only the API, shared packages + and its own plugin's files. What two plugins share otherwise lives in the base and reaches them through the API + (`src/runtime/schemas.ts`, `entries.ts`, `notices.ts`, `origin.ts`, `environments.ts`, `downloads.ts`, + `tui-chat.ts`). Base code imports no plugin code; only the page's shared components (`src/web/Drawer.tsx`, + `ModelPicker.tsx`) read plugin contract types (`import type`). A plugin's own files stay in the plugin, assets + included (`plugins/terminals/vendor/ghostty`). Facets use no `import.meta`: the p2 app directory is start data + (`Server.info.app`, `WorkerInit.app`). The terminal host (`plugins/terminals/main.ts`) is its own program, not a + facet, and imports base code directly. `plugin-layout.test.ts` builds every server, worker, TUI and web facet with + Chord v2's `buildFacet` against the API. +- **Runtime registries and locals** (`src/runtime`): `Routes` (HTTP by `METHOD /path`, exact path), `WorkerInitValues` + (spawn parameters contributed per plugin; a throw refuses the start, e.g. archived sessions), `Hosts` (worker + states, stop), `HarnessHost`/`HarnessNative`, `Extensions`, `HarnessConfig`, `Slots`, + `useObject`/`useReadable`/`useStatus`. +- **No configuration process globals.** Diagnostics alone have native process-scoped buffers because they capture + stdout/stderr before/after object activation and survive facet reload. Host configuration is the core's: `Server` (server: `info` = data dir, + fake, tunnel hosts, port; `revokeOwners`, `revokeGuests`), the `ServerNatives` (resource options, the session + workers, a root of the server's own) and `WorkerProcess` (worker: `init` spawn parameters, `liveness` counters, + `location` (where the session works now, see **Relocation**), a root of the worker's own). Code outside Chord calls + (Durable tools) runs in those roots; objects read registries with `self.registry(R).value`. The server + listens before loading plugins (so `info.port` is final); requests wait until the plugins are loaded. +- **Harness:** the core's `HarnessHost` opens the native Durable Harness and contributes it to `HarnessNative`; + dependents read it with `harnessOf(self)` (`.durable`). Its `environments` are + what Durable gets (`ExecutionEnvironments`): `session` is the session's location now, `locate(id, context, read?)` + a conversation's location (see **Agent locations**), `at(location)` the environment there (async: one machine, + i.e. connection, per environment name, opened lazily and shared by every conversation working there; one + environment per directory). `conversationEnvironment` does both; `locateConversation` locates from code that must + not hold the Harness (a prompt section, a tool). Resolve at each use; a relocation changes the session's location. +- **Location tools** (`plugins/locations`): `location_envs` (this device and the configured SSH environments: name, + host, last test; never keys), `location_dirs` (`[env:]path` or `env`+`cwd`, at most 200 folders) and `location_move`. + Every main agent has them; a subagent only when its main agent chooses them (`subagent spawn/configure tools`; not + in the default selection). All require a turn the owner started (`originOf`; a main agent's message to its subagent + carries the main turn's attribution as a `p2.subagent-task` entry), and the server side (`AgentLocations`) admits + only current worker generations. The main agent's move is deferred: it records `p2.location-move` on the main + conversation; the worker applies it through `SessionMoves` (the owner's Relocate) once Durable has no task or + submission (the turn ended, no subagent work), retries a busy refusal after the next commit, records a destination + that became invalid as `failed` (shown by `location_envs`), and applies a record left by a stopped worker after its + next start. A subagent's `location_move` changes only its own location, at once: its next tool calls run there. + `subagent spawn/configure` take `env` (null: this device) and `cwd` (relative to the caller's cwd on its + environment, to the home on another), and configure `followSession: true`; another environment needs an owner + turn; a location changes only while the subagent is idle ("… is working"). PowerShell is per agent: the deferred + `powershell` tool is offered (the agent's `tools`/`modelTools`) to each conversation whose location's machine runs + Windows and that has `bash`, re-decided after commits and moves. Tests: `locations.test.ts`, + `relocate-worker.test.ts` (PowerShell per agent), `powershell.test.ts`, `envs.test.ts` (listing an SSH env). +- **Agent locations:** every conversation works at a location `{ env, cwd }` (`src/runtime/location.ts`, browser-safe). + The main conversation works at the session's (the catalog's `env` and `cwd`); a subagent spawned without a cwd and + every one-shot agent follow it dynamically; a subagent spawned with a cwd stores its own location, `{ env: the + caller's environment, cwd }`, in its `p2.subagents` registry entry (`location`), fixed for its lifetime. The + subagents plugin serves `ConversationLocations` (`place`: migrate and list followers and own locations; `own`: one + conversation's own location), which the Harness consults for Durable's environment builder (by conversation ID, + not agent cwd), tools, prompt sections, attachments, workspace, file index and the server's terminals + (`Child.location`, from `ChildStore`). A relocation moves the main conversation and its followers only; subagents + with their own location keep their environment and cwd and stay usable on their machine. Migration: the registry + document is version 2; version 1's own `cwd` becomes `unplacedCwd`, which the worker places before its Harness is + live with the environment the server recorded (worker init value `subagentLocations`, from `ChildStore`), else the + session's spawn environment (version 1 refused to move a session to another machine while it had them). + `ChildStore` (state version 2) turns stored rows with an own `cwd` into `location` with the catalog session's + environment at its first read and saves them so. UI: the subagent bar ("· in gpu:/path" or "· follows the + session"), the sidebar row (own location), the spawn card, the workspace header and terminal tabs (title) show + `env:/path`; the TUI's `/agents` rows and spawn card likewise. Envs: deletion is refused and config edits restart + workers while any session or subagent location uses the environment (`sessionsUsing`). Sharing: read/write links + need the remote confirmation when any of the session's conversations works on a remote environment + (`remoteEnvironments`; `Sharing.env` lists them). +- **Relocation (in place):** `Sessions.relocate` (web Relocate, TUI `/cd`) does not restart a running worker. + - **Server** (`SessionWorkers.relocate`, serialized with starts, stops and pauses): fences the worker with + `WorkerBoundary.prepare` (refused while anything runs: "Stop the current run before changing directory", a named + subagent, background work), updates the catalog, then calls `WorkerBoundary.relocate`, which runs + `Harness.relocate` and lifts the fence. The fence counts as an active call, so the worker cannot stop idle + meanwhile. The steps run in a root of the server's own (`host.root`), not the caller's: an aborted call or closed + tab cannot cut them off. A worker is never left fenced: a failed catalog update lifts the fence (`cancel`); a + worker whose fence state is unknown (prepare failed after the fence, or cancel failed) is stopped. A worker that + exits during the steps (crash, idle stop racing the fence) gets only the catalog update. While a step calls the + worker (prepare, move, cancel), its route fails on a dead link instead of queueing a start behind the relocation, + which would wait for itself (`calling`); stops, restarts and catalog updates without the worker hold route + resolution, so arriving clients wait and then get a worker (`waiting`). A stopped worker only gets the catalog + update and starts at the new location. + - **Fallback:** if the worker cannot follow (its relocate fails, or takes longer than `moveTimeoutMs`, 60 s like + a start), the catalog is already moved: the server stops and restarts it, and the start reconciles the agent cwds + (its references go `gone` and are re-got). If that start fails too, the move still stands and the failure is the + worker's error status (Retry), not the call's. + - **Worker** (`Harness.relocate`), in this order: opens the destination's environment (a machine that cannot be + opened refuses before anything moves), switches the session's location (followers locate there from then on), + commits the root and its followers to the new cwd (the same step as at startup, after the placement provider + placed what version 1 left unplaced), sets `WorkerProcess.location` (`SessionLocation`: its listeners reopen + what they keep), then retires what nothing uses any more: environments at directories neither the session nor a + subagent's own location uses (their running commands killed), and machines without any (their connection + closed). A subagent's own location keeps its environment, on its machine. + - **Plugins** read `location.value` at each use instead of `init.values.session` (which stays the spawn + location). Between the switch and the commit, `location.value` still names the old place while + `Harness.environments` resolves on the new machine: whether p2's private data directory is denied is decided from + the environment (`isLocalEnvironment`), never from the location (workspace, file index, attachments). What is + kept for a location is keyed by `location.version` (bumped at each commit): prompt resources load anew from the + new cwd/machine (superseded loads publish nothing), workspace backends reopen their watch (subscribed before + their first open; a failed reopen is retried at the next use, superseded watches closed) and bump `changes`, the + file index drops its cache and builds in flight, dev-tools and PowerShell detect again. Chord resources reopen + only on reload, so these resources keep their value and swap what it holds inside. Clients keep every + reference: Chat, workspace, file index and prompt objects stay live. + - **Refused:** busy work (above), a destination that is not configured or not a directory, and whatever a + `RelocationChecks` contribution refuses under the fence (`prepare({ to })`; a refusal lifts the fence before the + catalog moves; none is contributed now). Subagents with their own location never refuse a move: they stay. + - **Unchanged:** cloning stops the worker (`pause`: the store is copied while closed; same fence handling; the + copy's catalog entry is read after the pause); idle stops; env-config edits (`envs` stops idle workers using an + edited environment, through the session or a subagent's own location, serialized after a move). + - **Tests:** `relocate.test.ts` (cold move, live references with bash/prompt templates/workspace at the new cwd + and the same worker generation, followers vs own location with their prompt templates, the restart fallback and + its failing start, a failing update/cancel/prepare, an exit while fencing, a client arriving meanwhile, a + cross-machine move accepted under the fence with an own location, a crash before the move, a move past its + bound, caller aborts, fences, a busy refusal leaving no fence), `relocate-worker.test.ts` (one worker host with + the real Harness: dev-tools and PowerShell after real moves, the private directory decided by the environment, a + move during workspace activation, a transient reopen failure, the session moving to another machine while a + subagent's own location stays with its tools, workspace and file index, concurrent commands on two machines with + one connection each and closing only unused machines), `harness-cwd.test.ts` (version 1 registry placement), + `envs.test.ts` (local to SSH and back in place, an own local location staying while the session moves to SSH, + real sshd). +- **Retries:** `retrying()` (src/runtime/retry.ts) only retries `upgrade_conflict` (an old root reaching an object a + reload migrated), in a fresh root. Roots of a worker that started behind the server are admitted by Chord. +- **Principals** (`src/runtime/principals.ts`, typed: `P2Principal`, with Chord's API typed by it in + `src/runtime/chord.ts`): `owner` (trusted loopback `access: "local"`, authenticated Radius `access: "relay"`, or a + signed owner cookie `access: "device"` with a device ID), `guest`, `worker` (stamped by the server's `session:` route + with the spawn generation), `server` (the server's own requests as a worker sees them, from the worker's `connect`) + and `host` (every host's own code, `createHost({ self })`). Object keys: `owner:local`, `owner:relay`, + `owner:device:<id>`, `guest:<id>:r<revision>`, `worker:<session>:<generation>`, `server`, `host`. Device IDs isolate + cookie holders so logout can revoke their live objects/links; "revoke all remote owners" is two propagating patterns + (`access: "relay"`, `access: "device"`). The owner's name on entries is settings.json `ownerName` (default "Owner"), + read by the worker (`WorkerModels`, hot-reloaded) when it writes an entry; entries keep the name they got. +- **Placement** (Chord's `host` on remote contracts, `src/runtime/chord.ts`): `serverHost` (the server) or + `sessionHost` (`session:` + the input's `session`). No caller names a host (no `at`): `get`, `hold`, `useObject` + and the TUI's `watch` go where the contract places the input. `WorkerBoundary`, `SessionTransfer` + and `PluginRemoval` (version 2) take a `session` input for that. Moving a type is a contract break. +- **Authors** are assigned by the worker from the principal and written as the first speaker block of every input + (`src/runtime/speaker.ts`); stop/dequeue restore text only to its author; removing a queued input (×, + no restore) discards it, as in Pi Mobile. +- **Windowed conversation view** (`plugins/chat/worker.ts`, `src/web/history.ts`, `src/runtime/conversation-stats.ts`): + `Chat.view` is Durable's view as is: a window over the transcript (the fork-aware history in entry ID order, head + markers in place), the newest 50 entries, then appends, back to 50 past 100, `before` the entry just before it. The + worker holds one Durable watch per conversation (default window), shared by every viewer. Clients (web and TUI) + keep a `TranscriptStore` per host and conversation over Durable's `ConversationTranscript`: every value of the view + is merged in, so entries that leave the window stay; scrolling up loads earlier pages (`Chat.entries`, 100 entries + or about 1 MB a page); a reconnect re-hydrates the shared window and its merge continues the transcript, or leaves a + gap when more was committed meanwhile than the window holds. There is no per-client `transcript.resume`: Chord + state is one cell per object, and the merge gives the same items. A gap is a "Load more" row styled as the top edge + (`button.history-edge.earlier`), filled a page at a time while it is rendered (the virtual list renders rows near + the viewport) and on a tap; in the TUI a muted "Load more" line, filled while it is in view. Reply quotes and search + hits load earlier pages or gaps until their entry is there (`loadThrough`). The active context's numbers (context + tokens, latest usage) are `Chat.stats`, kept in the worker by `ContextTally` (`src/runtime/conversation-stats.ts`) + from the entries the window appends: the latest counted response's reported usage plus pi-ai's estimate of what + follows it, about 4 µs per appended entry. Partials are `pi.live` changes and append no entry, so streaming costs + nothing here. It reads the whole context (`Conversation.context()`, a scan of every active entry: 13 to 18 ms at + 3,000 entries) only at the start, after a compaction, reset or edit, and when the window skipped entries. The cost + sheet reads `ChatCosts`, from `context()` when it opens. Measured with 3,000 seeded entries (fake provider, slayer): + opening sends 50 entries (125 KiB, 14 ms) instead of 101 (251 KiB, 33 ms); paging to the start 30 pages, 244 ms + (before: 29, 186 ms); building the whole transcript 1.3 ms (before: 10.9 ms). +- **Run failures** (`pi.live.runFailure`): why the newest run ended without an answer shows as a failure notice at the + end of the transcript (web and TUI, the error notice style) until the next run starts: `model_error` and `no_model` + "The model could not answer this input. <detail>", `faulted` "The run failed: <detail>", `orphaned` "The run could + not continue: <detail>", `interrupted` "The answer was interrupted: the server stopped while the model was + answering.", `restart_limit` "The run was stopped after repeated restarts. <detail>."; nothing for `aborted` (Stop) + and `reset`. A blocking compaction failure that ended the run shows once (its compaction notice). p2 no longer + writes `app.model-error` entries (they projected unanswered inputs before Durable had `runFailure`); stored ones + still render. +- **Sheet swipe** (`src/web/sheet-swipe.ts`, phones only, below 700px where sheets are bottom sheets): a downward + drag that starts on a sheet's top edge (a grip above its header, and the header; both `touch-action: none`) moves + it with the finger and fades its scrim; release closes it on a downward fling (> 0.3 px/ms) or past 30% of its + height, else it settles back; the route closes when the slide out ends. Drags in the content stay scrolls; upward + pulls give 20%. Both swipes share the drag mechanics (`src/web/drag-gesture.ts`: following one touch, deciding its + axis within 35 degrees and owning it with cancelable moves, release velocity, settle timing); each supplies where + it may start, which direction it takes, how it paints and what release means. +- **Arriving messages** (`useArriving` in `Chat.tsx`): rows appended after the previous last row while the transcript + grows fade in and rise 8px (0.28 s, 45 ms apart, at most 360 ms of delay; none with reduced motion). Not the first + rows, older history, rows replaced in place (a streaming answer committed, a sent message the server now shows) or + rows the virtual list mounts again. Opacity and transform only, so no layout or scroll change. While following the + answer, the transcript glides up by the appended rows' height (`#glide` in `use-chat-scroll.ts`: a 0.32 s transform + animation on `.messages`, at most a viewport) instead of jumping. A transcript appearing fades in (0.18 s), and large + spinners show only after 0.4 s, so a fast load (a kept conversation) never flashes one. All off with reduced motion. +- **Reopening the last conversation** (`resumeRoute` in `src/web/navigation.ts`): the installed app (standalone display + mode, iOS `navigator.standalone`) launching at `/` reopens the conversation open when it was last used (recorded on + navigation and when the page is hidden, `localStorage` `p2-last-conversation`) if that was less than 2 hours ago. + Going home forgets it; a browser tab opening `/` and a launch with another address are unaffected; signing out + forgets it. +- **Conversations kept on the device** (`src/web/conversation-cache.ts`, IndexedDB, the owner's browser only): the + transcript's newest entries without a gap among them (up to 4 MB) as a window (`before` the entry just before them), + with the documents but without the live run and queue, and the measured row heights, of the 20 most recently used + conversations, saved a second after changes and when the page is hidden. A page opening a conversation shows the + kept one at once (cold start, reload after an update) by seeding its `TranscriptStore`; the first live view is + merged in when it continues it (its `before` is one the kept entries reach), else the transcript starts over from + the live view (no gap), as it does when the kept entries are newer than the live view's newest (the server lost + them, e.g. restored from a backup). The database is version 2: what version 1 kept (a view and history pages) is + dropped once. Kept conversations of sessions missing from the owner's session list are forgotten whenever it + changes (`keepSessions`). Whether the browser was the owner is remembered in `localStorage` + (`p2-owner`), so the composer is there before the connection; signing out, or the server saying this browser is no + longer the owner, forgets both. +- **Service worker** (`public/sw.js`, served at `/sw.js` with `no-cache`): the app starts without the network once it + was opened. App pages come from the network first and fall back to the last app page kept (on failure, or after 3 + seconds); the web bundle's content-named files come from the cache first (the newest two bundle versions are kept); + everything else goes to the network untouched. Pi Mobile had none because it would have kept stale code across Vite + HMR and restarts; p2 serves no source modules (dev mode included), so every kept file is content-named and the page + never wins over the network. Signing out deletes its caches. +- **Sending never waits** (`plugins/chat/outbox.ts`): Send clears the composer and the message shows at once as it + will look (a person's message, or a queued input while a run is busy), whether or not the page is connected. A + per-conversation outbox sends in order whenever the conversation's chat object is live, each message with its own + request ID, so a retry after a lost answer is admitted once. Refusals (`forbidden`, `invalid`, a rejected upload) and + a send failing six times while connected put the text, attachments and reply target back into the composer with + the error; Stop takes back what was not sent yet. The outbox lives for the page: leaving with unsent messages asks + first. Who the page is (`useOwner`, a guest's grant) is kept while reconnecting, so the composer never turns into + the "Connecting…" bar after a drop; only a page that never connected shows it. +- **Virtual transcript** (`src/web/use-chat-scroll.ts`, `virtual-list.ts`, `row-estimate.ts`, `scroll-gesture.ts`; + after TanStack Virtual and react-virtuoso, without depending on either): the web transcript renders only the rows + near the viewport (one viewport of overscan each way, everything up to the first row near the list's start) between + two spacers rendered with the rows. Rows carry their spacing as padding, so a row's border box is all it adds. The + scroll position counts from the top, so content arriving below the reader (an answer streaming in, tool calls, the + composer growing) moves nothing and needs no correction; following the answer writes the end. Changes above the + reader (older pages, rows measured for the first time) are kept by the browser's scroll anchoring where it has it + (`overflow-anchor`, Chrome and Firefox; the spacers are never anchors), also mid-fling and under a finger. Without it + (Safari) the first visible row keeps its place: compared in content coordinates while the reader scrolls, on screen + otherwise; written to `scrollTop` when nobody scrolls, held as a transform of `.rows` while a finger is down, a fling + or the wheel's smooth scroll runs, written once the scroll settles or right away near the edge it would push content + past. Rendering a row at an end of the range can itself move the scroll position (scroll anchoring) and with it the + range; a range change that drops exactly the rows the last change added keeps them (`#updateRange`), so the list + settles instead of rendering every frame (`ui-virtual-scroll.test.ts`, fixture `edge-row.ts`). Measured heights are + kept per conversation (`keptSizes`, 8 conversations); rows never shown are estimated from + measured rows (`rowEstimates` in `Chat.tsx`: per row kind, a line over text length per width; images exact from + their headers, `image-size.ts`; one fallback height before anything is measured). Older pages load while the reader + is within 3 viewports of the loaded start, and the top edge (Earlier messages, a fading spinner, Start of + conversation) has one height in every state. Rows far from the viewport are not in the DOM: browser find-in-page and + tests see only rendered rows (`.rows[data-rows]` counts them); `jumpToEntry` reveals an entry through the list + (`revealInTranscripts`). +- **References** (Chord entry 8): an awaited `ctx.get`/`host.get` is ready with its state; UI runtimes hold + synchronously (`host.hold` in `useObject` and the TUI's `watch`) and read state through `useReadable` (undefined + until hydrated). Multi-variant references narrow on `variant` before using a member that not every variant has: + inline, `inVariant(ref, ...variants)` (the ref or `undefined`; a loading ref has no variant yet), `useObject(type, + input, { variant })`, or `expectVariant` where a mismatch is a bug (throws `not_in_variant`); all in + `src/runtime/chord.ts`. A TUI `watch` binds after `ready`, so its callback narrows on `ref.variant`. Tests use + `holdIn(host, type, input, variant)` (held, ready, `expectVariant`). Method-only references take `{ state: false }`. +- **Lingering views** (`UI_LINGER`, `src/runtime/refs.ts`): the web's `useObject` and the TUI's `TuiScope` release a + Chat reference 2 minutes after its view goes (at most 4), so switching back to a conversation binds at once; a + lingering reference counts as a viewer for worker liveness. Transcripts live per host and conversation (`transcriptOf`, at most 4 not + viewed), and a view taken again merges its snapshot into them. +- **Worker liveness**: viewers (live Chat objects and held workspace backends), active calls, Durable tasks; the host manager stops an idle worker + after `P2_IDLE_MS`. Browsers re-`get` a reference that went `gone` (`useObject`). +- **Conversation views** are mirrored with `cell.replace`: a new Durable revision shares unchanged containers with + the last, so subscribers get a diff. `Chat` exposes `Conversation.view` directly (a held reference's state member). + Durable watches become listener sources without memoizing: a reload keeps the subscription (same resource, same + `select` source text), so a watch is never started twice. +- **Sharing** (`plugins/sharing`): `Shares` (server local, `shares.sqlite`: links, guest identities with a + revision), `Presence` (connected guests, in memory), `GuestPresence` (held by a guest page's web facet; its + resource enters/leaves `Presence`), owner-only `Sharing` (links, connected guests, link/reset/remove). Guest + principals carry their grant revision; removal, link reset, archive and restore bump it and call + `invalidatePrincipal`, so the guest's objects go `gone` and its socket closes. The server consults the + `Authenticators` registry (guest cookie) and `SessionAccess` (guests reach only their session at the current + revision). Guest names are unique per session ("Alice (3)"), as in apps/mobile. The web share sheet is a + `session-actions` slot in the chat top bar, held in the navigation stack; Back closes it and reload restores it. + `test/p2/sharing.test.ts` covers guests over WebSocket; `ui-drawer.test.ts` a read-only guest page behind HTTPS. +- **Files** (`plugins/files`): `POST /upload?session=&name=` (owner or read/write guest of the session; exact + same-origin `Origin`; 25 MiB; stored in `<data>/uploads/<session>/`, never in the project) returns an attachment; + `Chat.send({ attachments: [ids] })` resolves IDs through the worker's `Attachments` local (only files of that + session), and the message lists their paths, with images inline when the model accepts them. `DownloadStore` + (server local) saves immutable snapshots for producers (send_file); `GET|HEAD + /download?session=&id=` serves them to the owner and the session's guests (no `Sec-Fetch-Site: cross-site`; + ranges for audio/video). Owns local `Uploads`, `Downloads` and `filePreview` implementations. Web: the composer's + attach button and drag/drop upload files at send time, including a new session's first prompt. +- **File index** (`plugins/file-index`): `FileIndex({ session, conversation })` (participants, as `Workspace`) answers + `@` completion (`complete`, ranked like pi's `@`) and Tab completion (`list`, one directory) for the TUI and the web + composer. A worker-local cache per conversation (`FileIndexCache`) builds from `WorkspaceBackend`'s `files` query + (git ls-files or the bounded walk, private paths excluded, `truncated` surfaced), derives directories from paths, + and rebuilds lazily when `changes` advanced (at most once a second). Cwd-relative only (`~`, absolute and `..` + complete nothing). Text handling shared by both UIs: `src/runtime/completion.ts`. p2-tui-port.md sections 8, 10.2. +- **Workspace** (`plugins/workspace`): owner/read-write-guest `Workspace({ session, conversation })` queries a + worker-local backend shared by that conversation's viewers. Owns its environment-backed workspace + reader, types/path/glob helpers, fuzzy filename ranking and `WorkspaceFile` (safe previews, inline diffs, + line comments, outdated quotes). The backend validates the conversation and reads `Harness.environment`, never + a server-local filesystem. Holds lease its `FileSystem.watch` (tree, HEAD, index, config); events increment + `changes`, coalesced over 100 ms. The last hold closes it; no heartbeat or 75-second lease. Compatible plugin + reloads keep the resource and listener. Liveness counts the held backend as a viewer, including while its watch + is opening (and until it finishes closing), so slow filesystem activation is not mistaken for idle time. + `GET|HEAD /workspace-file?session=&conversation=&path=` streams bounded worker byte chunks, including audio/video + ranges; `POST /workspace-upload?session=&conversation=&path=&name=` sends chunks to a temporary file (25 MiB, + exact same-origin Origin), then checks the destination is free before renaming. Worker upload writes serialize + across conversations to prevent competing uploads from overwriting each other; external filesystem races remain + the ExecutionEnv's confinement/rename limitation, not a sandbox. Every HTTP request rechecks current access; + read-only guests and cross-session guests are denied. Private data and `.git` paths cannot be opened. + Web: route-owned sheet with Files (tree, fuzzy filter, uploads), Changes (working tree, paged branch commits), + Search (literal text and include/exclude globs), File/Changes/Preview views, and Comments. Review drafts and open + editors persist in localStorage per conversation; Send/Send all use ordinary attributed Chat input. Integration + tests cover fake-provider prompts, a temp git repo, bounds, reload/watch lifetime, HTTP streaming/ranges, upload + cleanup/concurrency and guest revocation; `ui-workspace.test.ts` the browser panel. The web panel docks at 1200 px, + remembers its resizable list width, and shows a file beside the list when both chat and file can retain 480 px. + Read/write/edit cards supply one-step Open in workspace links. Files/commits push; views/search replace; tabs reset + the trailing workspace run. List/file scroll positions survive Back and reload. +- **Remote environments** (`plugins/envs`, [guide](remote-envs.md)): owner-only `Envs` lists environments with their + last test, saves/deletes/tests configs, accepts/forgets host keys and browses local/remote directories. Key generation, + public-key retrieval and private-key upload are available to every owner (including associated and Radius owners), + as in Pi Mobile, never guests (owner decision; formerly device-only). A proxy alone grants no ownership. Uploads + pass through the tunnel/relay, which must be trusted with the private key. `envs.json` has independent server/worker observers; invalid edits + retain the last valid state. Sessions persist `env` beside raw cwd and display `gpu:/path`; subagents with their own + location carry its `env` (see **Agent locations**). Deletion checks all sessions and subagent locations, archived + ones included. Valid edits stop idle workers using the environment; busy ones keep their captured environment until + idle. `HarnessEnvironments` supplies a lazy machine per environment (`open(env)`: environments at its directories + over one connection); missing/unreachable envs never use local tools. Attachments copy to remote + `~/.pi/p2/uploads/<session>/` of the machine where the receiving conversation works, before admission; inline + images retain the local upload copy. Project instructions, skills, workspace and downloads use the conversation's + environment. The local global AGENTS.md stays + local; remote skills use `~/.pi/p2/skills/`. New-session Browse directories and environment editors use route-owned + locators (Back/reload); read/write remote shares require explicit confirmation at the API and in the browser. + `envs.test.ts` uses a fake provider and disposable loopback sshd with private fixture keys/HOME (skips without sshd + or a daemon). It covers host trust, key storage, directories, no local fallback, tools, workspace, uploads, prompt + resources, downloads, terminal env selection, busy-edit/idle restart, archived deletion and share confirmation. +- **Terminals** (`plugins/terminals`): owner-only `Terminals({ session, view })` controls list/open/attach/close/resize/theme, + acknowledgements and explicit host restart. A resource holds one connection to the data directory's detached terminal + host through a host manager's `detachedSocketDriver`; stopping the manager detaches, never kills the host or its PTYs. + The host has its own process lock and ten-minute idle exit, owns local/SSH pi-env daemons, and keeps a headless + libghostty terminal per shell. Server and worker restarts leave programs running; reattach starts from a VT snapshot. + A one-use, 30-second channel grant from the held owner object admits `/terminal` (owner + normal Host/Origin checks). + Input/output bytes and ordered output metadata use this separate channel; incoming control JSON is refused. Slow + viewers stop after a 256 KiB unacknowledged window (output after the latest snapshot; the snapshot itself does not + count) and resnapshot once they catch up; socket backlogs over 8 MiB drop + the connection. Attaching checks the terminal belongs to the object's session; `open({ conversation })` fills cwd and + environment on the server, from the conversation's location (`ChildStore`'s `Child.location`, else the catalog). + New terminals in archived sessions are refused. Compatible plugin reloads keep the attachment resource. + Web: route-owned side sheet or resizable bottom panel, tabs, new/close/foreground-job confirmation, browser-native + selection/copy, mouse/touch scrolling, pastes, IME corrections and the remembered Ctrl/Alt/Esc/Tab/arrows/symbol key + row. Placement is remembered. The wasm, wrapper, renderer and input helper are independent copies under p2. + SSH configuration comes from the envs plugin and the conversation's location environment; the separate `terminal-sessions.json` + mapping is no longer read. Host keys use `ssh/known_hosts` and `pi-env-<name>` aliases. `envs.test.ts` exercises a + remote terminal against a disposable localhost SSH fixture; no real remote host is used. Windows PTYs remain unsupported. The workspace has an owner-only + Terminal tab through the terminals plugin's `workspace-terminal` slot. Side and bottom panels carry across a + session switch, showing that session's terminals without starting one. Only explicit terminal-button/tab taps + request auto-start; attachment updates replace the route, never push or reopen a later destination. + Outdated hosts show Restart (ends all terminals), but idle outdated hosts are not automatically replaced yet. + `terminals-host.test.ts` covers shells, disconnect snapshots, foreground jobs, themes and flood resync; + `terminals.test.ts` covers guest/control denial, one-use byte grants, session isolation, reload and server restart; + `ghostty.test.ts` and `terminal-input.test.ts` cover the copied browser primitives. PTY tests skip when no daemon build + exists. +- **Search** (`plugins/search`): `SearchIndex` (server local, `search.sqlite`: root/subagent user/assistant text in FTS5 and + one cursor per conversation; derived, dropped on a schema change). Each worker's `Indexer` (lifetime "host") runs a task that reads + the server's cursor (`SearchFeed`, worker principals only, session from the principal), scans each root/subagent + conversation's immutable history after it in pages of 200, and sends them; each call returns once the server + committed it, and the cursor sent stays put until the last page of a range, so an interrupted scan repeats the + range (idempotent by entry ID). It runs at start and after every commit adding entries (Durable + `observeCommits`), and retries after an error (server unreachable, reload) from a fresh cursor. The speaker and + peer-sender blocks are not indexed. Owner-only `Search.search({ query, archived })`: sessions whose title or cwd + contains the query (sidebar order) and up to 41 message hits (literal terms, all required) in the archive scope. + Search hits include conversation IDs, and a hit opens that conversation (a subagent's too); default titles derive from the first root user message, explicit titles + survive reindexing, and `updatedAt` advances monotonically with indexed activity. The sidebar search box (owner) filters sessions and + lists message hits. +- **Configuration** (`plugins/models`): `ModelConfig` (server local) owns p2's `Resources` observer + (settings.json, models.json, auth.json; hot reload) and contributes it to `ModelResources` (a native), plus `summary` (model + picker) and `snapshot` (settings, providers, credential metadata, errors; never secrets). Workers run their own + observer (`WorkerModels`). `startServer({ resources: { createModels } })` injects a fixture provider (tests). + Edit `settings.json` as a file; valid changes hot-reload and in-flight requests keep their configuration. + Set the default model and thinking level with **Make default for new sessions** in the model picker; there is no Settings editor UI. +- **Devices** (`plugins/devices`): `npm run owner-link [base-url]` calls the direct device-only loopback token path, + then prints a private `/owner#<token>` link. Base defaults to `P2_PUBLIC_URL` or `http://127.0.0.1:$PORT`; invitation + lifetime is ten minutes, single-use. The nonced sign-in page is also the stranger landing page; it immediately + scrubs the fragment, POSTs it, then redirects to `/`. HTTP/proxy URLs never carry the token. Signed per-device + cookies are HttpOnly, SameSite=Lax, Secure on a remote origin, with a 30-day browser/server expiry. The private + `owner-key` signs cookies; `owner-revoked.json` persists individual logout, and all-device logout rotates the key + and revokes live remote-owner objects/links through `Server.revokeOwners`. First signed-cookie startup rotates + the old exposed raw-key cookie once (`owner-cookie-version`); no legacy raw-cookie compatibility. The web host + exposes no filesystem paths; it serves only app pages, `public/` files and bundle files. Headless `owner-token.txt` + is replaced each startup. Cookie owners are remote (`access: "device"` + plus device ID), even when redeemed locally, so they cannot mint invitations. Device bootstrap privileges remain + inherent to the machine. Compatible reloads keep invitations; restarts invalidate them but keep cookies/revocations. + Tests use only temporary directories and port 0, with no real provider requests (`devices`, `proxy-marker`, `tui`, + and the fragment-link browser regression in `ui-parity.test.ts`). +- **Loopback owner cookie** (Pi Mobile's, `src/runtime/loopback-cookie.ts`): the local owner's browser gets + `p2_device_<port>` (HttpOnly, SameSite=Strict, secret in private `<data>/device-cookie`) when it loads an app page + (`/`, `/s/…`) on loopback. Cookieless loopback requests stay the device owner (TUI, owner-link, first page load), + except on a share's join page: only an owner by cookie or relay skips the form, so a fresh profile/incognito tab + opening `http://127.0.0.1:<port>/join/<token>` joins as a guest. A guest cookie wins over the cookieless fallback + for every request (pages, WebSocket, files); the loopback owner cookie wins over a guest cookie (the owner's + browser stays owner). Checked only on loopback. `loopback-join.test.ts` (HTTP and Chromium with fresh profiles). +- **Provider login** (`plugins/login`): `LoginService` (server local) runs its own `Logins` (provider-owned + api-key and OAuth flows); its events arrive through a listener and become the `login` state (status, the pending + prompt, notices; never credentials or answers). `ProviderLogin` admits every owner (local, device-associated, + or Radius), never guests, and adds the provider list. Web: Providers in the sidebar footer for every + owner, regardless of browser hostname. Remote flows automatically choose device-code when the provider offers + it; account selectors remain interactive. Browser-callback flows keep the provider's registered callback URL: + complete sign-in in the provider tab, then paste the authorization code or final redirect URL. Anthropic shows a + code; OpenAI ChatGPT may land on an unreachable localhost page, whose full address must be copied from the address + bar and pasted into Providers. The manual prompt explains that the page need not load. A device-code event with + `openBrowserOnConfirm: false` (a provider asking clients not to offer opening its page) shows the verification + address as text instead of the Open link (web), and the TUI offers no "open in browser" for it. No pi-ai changes or custom + callback URL. API keys and pasted codes pass through the tunnel/relay; use a trusted transport. Once no browser + holds the sheet, an unfinished login is cancelled. Tests use fake providers (login.test.ts and remote-login.test.ts). +- **Web slots**: `SlotList({ prefix })` renders every `prefix:<name>` slot (e.g. `sidebar-foot:providers`); `Sheet` is + the shared modal with scroll memory (Escape and the scrim close the whole modal stack). Shell-level + `workspace-panel` docks beside `main`; `workspace-terminal` is an optional terminals-plugin surface. +- **send_file** (`plugins/send-file`): owns its native tool, reading through the conversation's + execution environment and saving immutable private snapshots into the files plugin's download store directory. + Directory sources and files over 25 MiB fail. The existing authorized `/download` route serves snapshots even + after source deletion or plugin reload. `test/p2/send-file.test.ts` and the files integration tests cover it. + The artifacts plugin supplies the shared viewer. +- **Artifacts and viewer** (`plugins/artifacts`): owns its native artifact tool and immutable version + storage (2 MiB UTF-8). `ArtifactList` reads the conversation's latest-only `ArtifactDoc`, independently of + transcript paging, compaction and reset; it subscribes before catching up and validates the conversation during + activation. `/artifact?session=&name=&version=` rechecks authorization for owner/guest, rejects cross-site reads, + and serves the preview with a CSP sandbox (no same-origin privilege, forms, child frames or objects). Only + previews receive the diagnostics bridge; `download=1` returns the original source. HEAD describes the same + representation as GET. The chat-content slot supplies the shared transcript card contexts and explicit-click + file/artifact viewers; a top-bar counter opens a picker of every version, newest first. The viewer/picker lives in + named `Route.layers` (Back and sessionStorage restoration); picking an artifact pushes a viewer above its list. + HTML/SVG use a sandboxed iframe, Markdown/text render safely, + sent files use the conservative file preview. Nothing opens from a server update. `test/p2/artifacts.test.ts` + covers fake-provider publication, versions, independent index, reload/restart, CSP, HEAD, guest isolation and + byte limits; `ui-viewers-search.test.ts` the picker and viewers. +- **Conversations** (`plugins/conversations`): the root-only `conversations` tool lists active roots without + starting workers, sends server-attributed peer input, and reads the latest completed text answer (bounded scan, + no thinking, including old history). The tool fixes owner/peer origin at its calling assistant entry: later guest + input cannot change that grant, and unattributed reports do not inherit older owner access. `Conversations` + accepts only current worker generations; `PeerConversation` accepts only the parent's `{ kind: "server" }` + principal. The server brokers the sender's ID/title from the catalog, checks archive around startup/admission + and receipts, and scopes deduplication to sender + Durable tool task/call. Calls are short and cancellable; + cancellation after admission does not undo remote work. Final answers never auto-forward. Owner peer labels + navigate to roots (restoring archives); guests get plain labels. `test/p2/conversations.test.ts` covers list/send/ + answer, authenticated peer replies, restart deduplication, concurrent cross-sends, busy answers, follow-ups, + invalid destinations, reload, guest/forged attribution, child tool exclusion and inaccessible public ingress. +- **Subagents** (`plugins/subagents`): owns its protocol-independent Durable extension (persistent + anchor conversations and exactly-once background reporter tasks, model/thinking/tool validation). The worker + facet (an object of lifetime "host") contributes it to `Extensions` and catches up child metadata after relevant commits, retrying + acknowledged reports on errors/reload. Server `ChildStore` persists the list in `subagents.sqlite`; worker + generation fences late reports. Owner `Subagents` readers see every session; guests see only their own. Reading + the list does not start/hold a worker. The sidebar expands children (latest entry first), with working dots; + `/s/<id>/c/<conversation>` opens an independent chat, draft and model control. A parent link returns to the root. + Browser configuration synchronizes the subagent registry via `ConversationConfigured`. Main Stop leaves child + work running; direct child prompts do not report to the parent. `test/p2/subagents.test.ts` covers bad models, + configured spawn, one report, answer retrieval, configuration, independent history, reload during child work, + idle restart, guest reads and sidebar persistence across server restart. +- **Code mode** (`plugins/codemode`): the worker facet registers Durable's `codemode` extension + (`createCodemodeExtension`, tool `codemode`, callers `["model"]`, replay-safe) in `Extensions`. One sandbox per worker + (`CodemodeSandbox`, 256 MiB VM heap: `CODEMODE_MEMORY_LIMIT_BYTES`, kept constant because it is part of the VM + fingerprint a resumed script must match) is a native of `CodemodeRuntime`: a reload opens a new sandbox and drains + the old one (it closes once its running calls ended), so scripts running in it go on. Removing the plugin by a reload first ends its running calls (`plugins/codemode/calls.ts`: + each reports "Code mode was removed while this script ran." in its output, then is aborted through Durable's + `abortTask`, which ends its script and nested calls and settles it with an error), then closes the sandbox (VMs and + worker threads end); the turn goes on without the tool. A worker stop (`Liveness.stopping`) skips that step, so + the next worker replays the scripts. The extension reads the sandbox + through a getter when a script starts. At worker shutdown the sandbox closes while the Harness is + still open (Chord disposes holders first); Durable stores no outcome for a run the sandbox ended on its own, so the + next worker replays the script. `codemode.test.ts` covers a script calling two tools (and its `details.calls`), live nested slots, a + reload during a script, SIGKILL and graceful restarts mid-call (finished calls do not rerun; the interrupted unsafe + `bash` call returns an error the script catches); the removal mid-script (prompt error, turn ends, the script's thread goes); `codemode-plugin.test.ts` the + sandbox across reload/removal. +- **One-shot agents** (`plugins/agent`, port of Pi Mobile a9335ebea and 0e238b29f): code mode scripts' `tools.agent({ + prompt, schema, label?, model?, thinkingLevel?, tools? })` (`callers: ["tools"]`, replay-safe). One conversation per + call, owned by the tool task (abort cascades) and recorded in the task-scoped `p2.agent-run` doc, so a rerun + reattaches; it does so before validating the model, so a model removed from the configuration since does not fail + it. Submissions use `agent:<attempt>` request IDs. Workflow patterns with example scripts: + [workflows.md](workflows.md); `workflows.test.ts` runs each of its scripts in fake mode (real model names replaced + by the fake one) and checks its claim that a failed `tools.bash` rejects with `structuredOutput`. The answer is parsed as JSON (a fenced block is accepted) and checked + with pi-schema's `compile` (formats checked, as TypeBox's compiler did) against `schema`; failures go back to the + same conversation, up to 3 attempts. The result's + `structuredOutput` is the value, `details` `{ conversationId, attempts }`; nothing is reported to the parent. It has no + prompt section (as Pi Mobile 1b763d912): its description, Pi Mobile's verbatim, carries the usage notes, and code + mode's section lists it with the other tools scripts call. Model/thinking/tool + validation is the subagent tool's `resolveChoice`. The plugin contributes the tool unconditionally and does not + depend on code mode: without a script nothing can call it (`callers: ["tools"]`), so code mode stays removable while + it is loaded (code mode's listing then disappears; `agent.test.ts` removes and re-adds code mode). It depends on subagents, + whose document lists its conversations. The creating commit + also writes the agent into the `p2.subagents` doc's `oneShots`, keyed by its nested call ID + (`<script call ID>/c<n>`); the subagents worker reports them as `Child` rows with `kind: "agent"`, `callId`, the + label as name and `lastEntryId: 0`. Their entries do not wake the report task (no ChildStore write per commit; only + the sidebar orders by activity). Read-only: the subagents plugin contributes `ReadOnlyConversations` + (`src/runtime/worker.ts`), which `Chat.send`/`dequeue`/`configure` consult (`forbidden`, "One-shot agents are + read-only"); `stop` stays allowed, and `SubagentControl.archive` refuses them. Web: the sidebar skips them; the code + mode card's nested `agent` row is a button opening the conversation (`OneShotAgents` context, `src/web/Chat.tsx`); + the child view shows the `agent` tag and a subagent bar ("One-shot agent of a script · read-only · Tools: …") with + Stop while it works, the scroll-to-bottom button (otherwise carried by the composer), and no composer or model chip. The TUI's nested rows show the label (else the prompt). + `agent.test.ts` covers parallel agents, retries, no reports, listing and read-only refusals; `ui-agents.test.ts` the + row button, the read-only view, scroll-to-bottom and Stop. +- **Subagent archiving:** `archived` on a `p2.subagents` agent, set by the tool's `archive`/`restore` or the + browser through the worker's `SubagentControl({ session }).archive({ conversation, archived })` (owner and read/write + guests). Archiving aborts the conversation first (`setArchived`, shared by both); `send` clears the flag; spawning an + archived name says so. `status` omits archived agents unless named and adds "N archived (status with a name shows + one)." `Child.archived` reaches the sidebar (`ChildStore`): archived + children sit behind an **N archived** row per session, open while viewing one. The subagent view now has mobile's + bar (Tools, Archive/Restore). The TUI lists no subagents, so archived ones need nothing there. + `subagents.test.ts` covers tool archive, status, browser restore/archive-with-abort, send restoring and guest + refusal; `ui-agents.test.ts` the bar and the sidebar row. +- **Subagent cwd** (port of Pi Mobile 24e3fccdd; see **Agent locations**): `spawn` takes `cwd` (description and errors + verbatim), an existing directory resolved relative to the caller's cwd through the caller's environment + (`absolutePath`, `canonicalPath`, `fileInfo` must be a directory; else "cwd X is not an existing directory."), + configured as the child's `pi.agent` cwd and stored as `location: { env: the caller's environment, cwd }` on its + `p2.subagents` agent (fixed; any other action with a cwd: "Only spawn takes a cwd."). Spawn details carry `cwd` + and, on a remote environment, `env`; `status` shows "· in env:/path"; `Child.location` reaches the sidebar list + (`ChildStore`). The Harness (`src/runtime/worker.ts`) gives Durable each conversation's environment at its location + (`Harness.environments`: one machine per environment, one environment per directory, all cleaned up with the + session's). `WorkspaceBackend` opens at the viewed conversation's location, and uploads write through + `WorkspaceWrites.write({ location, upload })` (one workspace per environment, still serialized). The watch locates + the git directory with `git rev-parse --git-dir --git-common-dir`, so a linked worktree's index/HEAD are watched. + With Pi Mobile 78056b34e: at every worker start and every in-place relocation the Harness moves the root and every + conversation that follows the session whose `pi.agent` cwd differs to the session's cwd, in one commit, before it + is live (so nothing runs or resumes in a stale cwd). The followers come from the `ConversationLocations` local + (`src/runtime/worker.ts`), served lazily by the subagents plugin (`placeSubagents`: subagents without a registry + `location`, one-shot agents) and without a Harness dependency (it gets the opening Durable Harness as an + argument). No timing assumption: the worker passes `serveHarness` a promise resolved once `host.load` has + installed every plugin, the Harness awaits it, then gets the provider and awaits its activation; an unserved type + (`unknown_type`) then means no followers and no own locations. Nothing resolves against the process cwd. Web: the + subagent bar adds " · in <location>", the spawn card "cwd <relative>" (`env:/path` remotely), and the transcript's + paths and the workspace panel use the child's cwd. TUI: the card's configuration line and `/agents` rows show it. + Tests: `subagents.test.ts` (missing/file/relative cwd, listing, status, bash pwd, per-conversation workspace list + and upload), `relocate.test.ts` (following vs own after a move), `subagent-cwd.test.ts` (a stale foreign cwd fixed + on restart), `harness-cwd.test.ts` (a provider installed late and activating slowly still moves followers before + the Harness is live; version 1 own cwds placed; without a provider only the root moves), `envs.test.ts` (real sshd: + a cwd-less subagent's bash in the remote cwd, after moving local and back; an own local location kept), + `workspace-environment.test.ts` (linked worktree index watch), `ui-agents.test.ts` (bar, card, sidebar row, + workspace panel). +- **Code mode setting (owner decision, differs from Pi Mobile):** `settings.json` accepts Pi Mobile's `codemode` + boolean, but p2 defaults it to on (mobile: off). `false` uninstalls the `codemode` extension live in every worker + without unloading the plugin: the worker's `WorkerModels` exposes its current settings, and the codemode facet + registers a switch in `ExtensionSwitches` (`src/runtime/durable.ts`), which leaves switched-off extensions out of + the Durable registry snapshots (a running script keeps its pinned tool). The `agent` tool stays installed (Pi Mobile + uninstalls it with code mode); it is described only in code mode's listing. Durable 8bb180bec's code mode (prompt section + listing callable tools, `models.*` through internal `pi.codemode.*` tools, coding-agent result layout) uses its + defaults (`inlineBudget` 3000, `models` on, no namespaces). p2 lists tools only from the subagents' fixed coding-tool + set, so internal tools never appear; nested-call rows show them by their script name (`models.classify`). + `codemode-setting.test.ts` toggles it live; `durable-registry.test.ts` covers switches. +- **Code mode card** (copied from Pi Mobile, `src/web/transcript.ts`, `Chat.tsx`, `icons.tsx`, `style.css`): the + `codemode` card shows `<>` Code with the script's first line; its body the script (highlighted JS, clipped to six + lines with Show all) and the output. `ToolView.calls` is the one source of nested-call rows: live from + `pi.live.nestedTools` (grouped by the model-issued call, depth from the `parentCallId` chain), settled from the + result's `details.calls` (Durable's `CodemodeToolDetails`). Rows show status, name, one-line arguments, duration and + error or last output line; the newest eight while running, collapsed to "N tool calls · k failed" once settled. + The TUI (`src/tui/messages.ts`) shows the first script line and indented rows (summary once settled, all with + Ctrl+O) until the TUI port replaces it. `codemode-card.test.ts` covers grouping/depth, the details mapping, the TUI + and the browser card (fixture `test/fixtures/codemode-card.ts`). +- **Security limits (as Pi Mobile):** for static UI requests, guests may load their session's pages, the installable + app's `public/` files, the bundle manifest and files from the current or briefly retained bundle generations + (`src/hosts/web.ts`); other web-host paths are 403. Authorized plugin routes retain their own guest permissions. + Raw filesystem source paths are never served directly; bundle source maps are available as bundle files. `/chord` + sockets (`src/hosts/socket-limits.ts`): frames up to 1 MiB (Chord's part limit too), no limit on calls in progress (owner decision: + Chord v2 has none), more than 4 MiB unsent output beyond the frame being sent + terminates it, socket errors never crash the server. Terminal sockets keep their own 1 MiB frames and 8 MiB + backlog. A provider login belongs to the owner principal that started it: only it sees, answers, cancels or + dismisses it, and its leaving (all holds of its Providers sheet gone) ends it; another owner device cannot. +- **Reuse** of apps/mobile: none. Adapted implementations live beside their plugins; shared backend helpers live + in `src/runtime`, shared browser code in `src/web`. p2 is an independent npm workspace with pinned dependencies. + `src/web/style.css` is a copy of mobile's stylesheet; terminal and environment styles live beside their plugins. + Web tests verify guest access to bundle files and rejection of raw backend, mobile and private-data paths. +- **Pi Mobile UI parity:** the web UI copies Pi Mobile's components, markup and styles; native Chord + locators/references, leases, permission checks and byte channels replace mobile's transport (no mobile imports, no + protocol compatibility client). Not exercised against live services: real terminal programs/SSH without a + packaged/local pi-env daemon build (offline terminal layout tests do not prove shell integration), external provider + sign-in, physical device sign-in/clipboard and Radius. Parity comparisons (390×844, 1280×900) covered representative + screens, not every provider/tool/media/error combination. +- **Voice mode** (`plugins/voice`, `voice-server/`; README "Voice mode"): hands-free talk with a conversation, like + Claude's voice mode; the screen stays on and audio goes through Bluetooth (no Android Auto app). Owner decisions: + speech runs on slayer's CPU (no GPU there): Parakeet TDT 0.6B v3 (int8, sherpa-onnx; about 6% of real time on the + i9-13900) and Supertonic 3 (picked over Kokoro, Piper and Supertonic 2 by ear; voice selectable, F1 default); English + only; no full-screen mode, an orb in the composer. Design: the speech service is a separate Python service (its + dependencies stay out of the npm tree) that p2 proxies for the owner, so the phone's p2 and slayer's both use it and + the page's CSP stays `connect-src 'self'`. The browser does the conversation logic with the ordinary `Chat` + contract: a final transcript is `send` (steering while busy), new answers (keys present when voice mode started are + skipped) stream through `SpeechChunker` (sentences, no code blocks or tables) to `/voice/speak`, played in order. + While audio plays the microphone is not streamed; pibot's barge-in detector (energy the playback does not explain) + only starts streaming it. Owner rule: only words interrupt. The speech service sends interim transcripts + (sts-server `interim`); the first with a real word (no filler, not the agent's own speech heard back) cuts the speech + and stops the run at once, also while the agent works silently (`interruption.ts`); a trigger the room never hears as + words is dropped and the agent goes on. Transcripts without words are never sent; a stop phrase also calls `stop`. A `voice.mode` conversation document turns the `voice` prompt + section on (short spoken answers); the switch follows the button. Tests: `voice.test.ts`, `voice-ui.test.ts` + (Chromium fake microphone). diff --git a/apps/p2/docs/p2-tui-port.md b/apps/p2/docs/p2-tui-port.md new file mode 100644 index 000000000..bd2deabc7 --- /dev/null +++ b/apps/p2/docs/p2-tui-port.md @@ -0,0 +1,1732 @@ +# Porting pi's interactive TUI to p2 + +Goal: when an existing pi user opens p2's TUI (`npm run tui`), it should look and behave like pi's coding agent +(`packages/coding-agent`, interactive mode), wherever that makes sense for a server-backed, multi-user, sandboxed app. +This is not a feature-parity checklist. The guide lists every user-facing feature of pi's TUI, says how p2 gets each +one, and gives the designs and the work plan for the port. Sections 1–9 are the porting design; where the build +differs, the "As built" sections of section 10 describe the code. + +Notation: + +- **(a) copy**: copy the source into p2 (usually `apps/p2/src/tui/` or a plugin's `tui.ts`) and strip what section 11 + lists. p2 must not import `packages/coding-agent` at runtime. `@earendil-works/pi-tui` as a library is fine. +- **(b) Chord**: re-implement on Chord objects. The table names the object or contract that supplies the data, + marked **new** when it does not exist yet. +- **(c) skip**: not applicable, with the reason. **(d) deferred**: applicable, scheduled late. +- **L:** a launcher action under a launcher grant (section 3). **G:** a guest-local preference (section 4). + +Path aliases. Every pointer starts with one of these or a repo-root path: + +| Alias | Path | +| --- | --- | +| `CA` | `packages/coding-agent/src` | +| `IM` | `packages/coding-agent/src/modes/interactive/interactive-mode.ts` | +| `CMP` | `packages/coding-agent/src/modes/interactive/components` | +| `THM` | `packages/coding-agent/src/modes/interactive/theme` | +| `REN` | `packages/coding-agent/src/core/tools/renderers` | +| `TUI` | `packages/tui/src` | +| `ENV` | `packages/env` | +| `DUR` | `packages/durable/src` | +| `P2` | `apps/p2` | + +`IM`, `CMP`, `THM`, `REN` and `TUI` line numbers refer to coding-agent and pi-tui as of the port; p2 code is named by +file and symbol. Section 2 maps where the port lives in p2. + +--- + +## 1. Feature inventory and how p2 gets each + +### 1.1 Layout, header, footer, renderer + +| Feature | coding-agent pointer | p2 route | +| --- | --- | --- | +| Two renderers: regular (main screen, scrollback) and fullscreen (alt screen with own scroll, search, selection), switchable at runtime | `IM:873` mountInteractiveTui, `IM:890` switchTuiMode, `CA/modes/interactive/tui-renderer.ts:22` | Fullscreen only, as a **product decision**. Regular mode is technically possible: the filter passes text, cursor and erase sequences (`P2/src/hosts/tui-filter.ts:5-13`, `:49`). But in regular mode a worker swap (reload) re-renders the transcript below the old scrollback. That duplicates content unless the new worker restores `TuiMainScreen.captureRenderState` from the snapshot. If regular mode comes later, the reload state must carry that render state. | +| Fullscreen layout: transcript `ScrollView` above a fixed dock (pending, status, widgets, editor, widgets, footer) | `CA/modes/interactive/chat-viewport.ts:22-45`, `IM:968-991` | (a) copy `CA/modes/interactive/chat-viewport.ts` (as built: `ChatSurface.layout`, `P2/plugins/chat/tui.ts`). | +| Transcript scroll keys (page, half page, line, top/bottom, prompt jumps), "Jump to latest" indicator | `TUI/keybindings.ts:160-210`, `CA/modes/interactive/tui-renderer.ts:30-34` | (a) Use pi-tui's built-in keys. | +| Transcript search | `TUI/tui-alt-screen.ts:168` options, `CA/modes/interactive/tui-renderer.ts:25-29` | (a) Free with `TuiAltScreen`; pass themed styles. Key precedence: section 6.2. | +| Mouse: wheel, selection, copy-on-select, right-click paste, clickable logo | `CA/modes/interactive/tui-renderer.ts:35-46`, `IM:256` | Wheel and selection: (b), behind a **G** mouse policy plus launcher mode ownership (section 3.6). Copy is **L:copy**; right-click paste is **L:paste**. Easter eggs are (c). | +| Startup header: logo, version, key hints (Ctrl+O toggles compact/expanded), onboarding line | `IM:1009-1075`, `CMP/pi-logo.ts:19` | (a) Hints come from the command registry (section 6). Show the server origin and the role (owner/guest) in place of the version. | +| Loaded-resources listing | `IM:1772` | (d) Owner only: **new** resources view (AGENTS.md and skills from `P2/plugins/prompt/worker.ts`, enabled plugins). | +| Changelog / update notices | `IM:843`, `IM:4647`, `IM:4679` | (c) Server releases are not pi releases. Reuse the slot for owner notices. | +| Footer line 1: cwd, git branch, session name | `CMP/footer.ts:165-177` (session name `:174-177`), rendered at `:286` | (b) `Sessions.sessions` gives title, env and cwd. The branch comes from `Workspace.query({kind:"status"})`, participant only. Guests in view mode get no branch. | +| Footer line 2: tokens, cache rate, cost, context %, model and thinking | `CMP/footer.ts:179-212` stats, `:220-284` model side | (b) Data provenance in section 1.16. | +| Footer line 3: extension statuses | `CMP/footer.ts:290-299` | (b) `TuiFooter` registry (section 7). | +| Terminal title | `IM:1132` | **L:title** (section 3). | +| Progress / program status | `CA/modes/interactive/program-status-reporter.ts:17`, `TUI/terminal.ts:571` setTitle, `:576-584` setProgramStatus, `:587-600` setProgress | **L:progress**, gated by **G** `terminal.showTerminalProgress`. | + +### 1.2 Editor and input + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| Multi-line editor: kill ring, undo, word motion, jump-to-char, newline | `TUI/components/editor.ts:297`, `TUI/keybindings.ts:71-158` | (a) Already used. | +| Mouse editing / cursor placement in the editor | `TUI/components/editor.ts:655-693` | (b) Requires mouse mode (section 3.6). | +| `CustomEditor`: `onAction`, Escape/Ctrl+D handlers, paste-image hook, extension shortcuts, embedded working status | `CMP/custom-editor.ts:13`, `:84`, `:90-140` | (a) Copy, wired to the command dispatcher (section 6). The paste-image hook is removed: that key is launcher-owned. | +| Border color by thinking level; bash-mode color for `!` | `IM:4528`, `IM:3125` | (a) | +| Prompt history | `TUI/components/editor.ts:345-490` | (a) Seed it from the transcript's own user items, like `populateHistory` (`IM:3975`). Secret prompts never enter history (1.8). | +| Paste markers `[paste #1 +N lines]` | `TUI/components/editor.ts:1309`, `:1092` | (a) Free; bracketed paste is allowed. | +| Slash autocomplete with argument completion | `IM:725`, `TUI/autocomplete.ts:314` | (b) Fed by `TuiCommands` (section 6). Never instantiate `CombinedAutocompleteProvider`: it calls `readdirSync`/`statSync`/`spawn` (`TUI/autocomplete.ts:1-3`). | +| `@file` fuzzy completion | `TUI/autocomplete.ts:149-240`, `:772`, `:503` | (b) **new** `FileIndex` (section 8). | +| Tab path completion (`~`, absolute, parent) | `TUI/autocomplete.ts:520`, `:549-588` (homedir/statSync), `:600-680`, `:850-859` | (b) Cwd-relative only in p2 (section 8.2). | +| Extension autocomplete providers | `IM:825` | (b) `TuiCompletions`, keyed by provider id (section 7). | +| Ctrl+V clipboard paste (files, image, text) | `IM:3152`, `CA/utils/clipboard-image.ts:200` | **L:pasteImage**, launcher-matched (section 3). | +| Drag & drop files | `IM:1047` (the terminal pastes paths) | **L:attach** with trusted consent (section 3.3). | +| External editor | `IM:4608`, `CA/modes/interactive/external-editor.ts:14-46` (`editInExternalEditor`), `CA/core/settings-manager.ts:946` | **L:externalEditor**, launcher-matched; the command is **G** and never sent to the worker. | +| Ctrl+C clear / double Ctrl+C exit / Ctrl+D exit on empty editor | `IM:4281`, `IM:4291`, `CMP/custom-editor.ts:118` | (a) As pi (10.1). | +| Ctrl+Z suspend | `IM:4450` | **L:suspend**, launcher-matched. | +| `!cmd` / `!!cmd` user bash | `IM:3346-3360`, `IM:6999`, `CMP/bash-execution.ts:21` | (d) **new** `Chat.shell` (participant, streamed through a live slot or an entry). | + +### 1.3 Message rendering + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| User message (Markdown on `userMessageBg`; OSC 133 zone markers) | `CMP/user-message.ts:13`, `:49-50` | (a) Plus p2 data from `buildTranscript` items (`P2/src/web/transcript.ts`): author label with the guest "unverified name" note, peer attribution, reply quote, plugin note, files and image count. | +| Assistant message: ordered text/thinking, hidden-thinking label, error/abort/length notices | `CMP/assistant-message.ts:14`, `:58`, `:91`, `:142-168` (thinking blocks), `:185-198` | (a) `AssistantMessageComponent` (`P2/src/tui/components/messages.ts`), streamed with `updateContent(message, true)` by `AssistantView` (`P2/src/tui/components/transcript.ts`). | +| Per-thinking-block and per-card expansion (click) | `CMP/assistant-message.ts:142-168`, `CMP/tool-execution.ts:162-167` | (b) With mouse mode; otherwise global Ctrl+T/Ctrl+O. | +| Markdown + syntax highlighting; code-block indent | `THM/theme.ts:1022` getMarkdownTheme, `THM/theme.ts:930` highlightCode | (a) Port highlighting to `lowlight` (already a p2 dep, `P2/src/web/Code.tsx`). `markdown.codeBlockIndent` is **G**. Watch the 8 MiB bundle cap (`P2/src/hosts/tui-sandbox.ts:3`). | +| Mermaid | `CMP/mermaid.ts:1-5`, `:60-88` | (d) Pure rendering (grok-mermaid). Mode `markdown.mermaid` is **G**. Lands as an ordered Markdown transformer (section 7). | +| Skill invocation block | `CMP/skill-invocation-message.ts:11`, `IM:3945-3953` | (d) With `/skill:` (1.10). | +| Custom messages / entries | `CMP/custom-message.ts:12`, `CMP/custom-entry.ts:11`, `IM:3858` | (b) `TuiEntryRenderers` by entry kind (section 7). | +| Compaction summary | `CMP/compaction-summary-message.ts:10` | (a) Data: the `pi.compaction` entry. | +| Branch summary | `CMP/branch-summary-message.ts:10` | (c) Until p2 has branch navigation. | +| Reset marker | — | (a) `pi.reset` plus its handoff text (`MarkerView`, `P2/src/tui/components/transcript.ts`). | +| Cache-miss / warming / thinking-dropped / compaction-cost notices | `IM:4122-4210` | (b) Cost notices from `pi.usage`. Cache warming is server state (1.14). | +| Status lines (consecutive lines collapse), errors, warnings | `IM:3836`, `IM:4635`, `IM:4641` | (a) `StatusLines` in the dock (`P2/src/tui/components/chrome.ts`; 10.1). | +| Nested codemode calls | `IM:3607` | (b) `docs["p2.nested-calls"]` (`P2/plugins/chat/contract.ts:182`). | + +### 1.4 Tool call/result rendering + +The generic card is `CMP/tool-execution.ts:31`: + +- Backgrounds for pending/success/error, set in `updateDisplay` (`CMP/tool-execution.ts:270-276`). +- Fallbacks (`CMP/tool-execution.ts:142-159`). +- Expansion (`CMP/tool-execution.ts:202`) and the render context (`CMP/tool-execution.ts:120`). +- Lifecycle methods `updateArgs`, `markExecutionStarted`, `setArgsComplete` and `updateResult` (`CMP/tool-execution.ts:170-187`). +- Images (`CMP/tool-execution.ts:354`). + +Renderers are combined at `REN/index.ts:32`. + +| Tool | Pointer | p2 route | +| --- | --- | --- | +| Generic fallback | `CMP/tool-execution.ts:23`, `:142-159` | (a) Section 11 checklist. | +| `bash`/`powershell` | `REN/bash.ts`, `CMP/visual-truncate.ts:69` | (a) Map Durable details (`DUR/tools/bash.ts:130` `fullOutputPath`). | +| `read` | `REN/read.ts` | (a) Strip `getReadmePath` and local path resolution (section 11). | +| `write` | `REN/write.ts` | (a) | +| `edit` (word diff; pre-execution preview) | `REN/edit.ts:173-252`, `CMP/diff.ts:79` | (a) `renderDiff` needs the `diff` package, pinned. Drop the local preview (`computeEditsDiff` reads the local fs, `REN/edit.ts:189`). Use `details.diff` (`DUR/tools/edit.ts:127-137`). | +| `grep`/`find`/`ls` | `REN/grep.ts`, `REN/find.ts`, `REN/ls.ts` | (a) For subagent/MCP tools; Durable's coding tools are read/write/edit/bash. | +| p2 tools: artifact, send_file, subagent, ports, reload_plugins | web `P2/src/web/Chat.tsx:164-302` | (b) Plugin renderers (section 7). | +| Images | `CMP/tool-execution.ts:354`, `TUI/components/image.ts:103` | (d) Text placeholders now (`P2/src/hosts/tui-worker-runtime.ts:103`). A later launcher-rendered image is described in section 3. | +| Expand all (Ctrl+O) | `IM:4569-4590` | (a) | + +### 1.5 Working indicators + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| Spinner in the editor border; `setWorkingMessage/Visible/Indicator` | `CMP/status-indicator.ts:9`, `:38`, `IM:2346-2380` | (a) Text derived from `pi.live` (`ChatSurface.syncIndicator`, `P2/plugins/chat/tui.ts`; indicators in `P2/src/tui/components/status.ts`). | +| Retry countdown (Esc aborts) | `CMP/status-indicator.ts:51`, `CMP/countdown-timer.ts:7`, `IM:3744` | (a)+(b) `live.generation.retry`; Esc runs `Chat.stop`. | +| Compaction indicator (Esc aborts) | `CMP/status-indicator.ts:85`, `IM:3672` | (a)+(b) Needs **new** `Chat.abortCompaction` (P3 contract before the P4 UI). | +| Hidden-thinking label, idle status | `IM:2382`, `CMP/status-indicator.ts:114` | (a) | + +### 1.6 Themes + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| `Theme` class | `THM/theme.ts:185-373` | (a) Instance design (section 5). Do not copy `THM/theme.ts:375-391` (built-ins read from disk). | +| Tokens, schema, built-ins, system theme | `THM/theme-tokens.ts:24`, `THM/theme-schema.ts:52`, `:97`, `THM/dark.json`, `THM/light.json`, `THM/system-theme.ts:465` | (a) Built-ins are imported as JSON into the bundle. | +| Terminal color query and scheme notifications | `THM/theme-controller.ts:31`, `TUI/tui.ts:1494`, `:961` | **L:terminalColors** (the launcher queries; the worker gets data). | +| Custom theme discovery and loading | `THM/theme.ts:436` getCustomThemeInfos, `THM/theme.ts:487` loadThemeJson, `THM/theme.ts:734` startThemeWatcher; `THM/theme.ts:672` registers already-loaded themes into `registeredThemes` (declared `:670`); `CA/config.ts:614` | **G** The launcher reads and watches the files and posts validated JSON. | +| Theme picker with preview; light/dark pair | `CMP/theme-selector.ts:13`, `THM/theme.ts:578` | (a) Saving goes through **L:savePrefs** with trusted confirmation. | + +### 1.7 Models and thinking + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| Model selector: search, all/scoped toggle (Tab), current and default badges, Ctrl+S make default, background catalog refresh with progress and timeout | `CMP/model-selector.ts:40`, `:94`, `:137-158`, `:184-218`, `:262-296`, `:362-403`; `IM:5354`, `IM:5206` | (a) Section 11 checklist: inject data and actions instead of `ModelRuntime`. Data: `Models.models` and `Models.defaultModel`. "Refresh" uses `ProviderLogin.refresh` per provider. Owner only: `Models` refuses non-owners (`P2/plugins/models/server.ts:62-64`) and `Chat.configure` is owner-only (`P2/plugins/chat/worker.ts:269-272`). Guests see the model read-only. | +| Exact `/model provider/id` | `IM:5229` | (a) | +| Scoped models, Ctrl+P cycling | `CMP/scoped-models-selector.ts:97`, `IM:5390`, `IM:4550` | (b) **new** `enabledModels` in `P2/src/runtime/settings.ts`, exposed by `Models` (contract in P3, UI in P4). | +| Thinking selector, Shift+Tab, save as default | `CMP/thinking-selector.ts:36`, `IM:4539`, `IM:5185` | (a) `Chat.configure` / `Models.setDefault`; owner only. | +| Per-model default thinking (`modelThinkingLevels`) | `packages/coding-agent/docs/settings.md:16` | (b) Server setting; read-only display. | + +### 1.8 Login / logout + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| `/login [provider]`: auth type and provider selectors | `IM:5863`, `IM:5897`, `CMP/oauth-selector.ts:58` | (a) Data: `ProviderLogin.providers` (`P2/plugins/login/contract.ts`), owner only. | +| OAuth dialog: URL, device code, manual code, progress, cancel | `CMP/login-dialog.ts:12`, `CMP/auth-url.ts:10`, `IM:6290-6298`, `IM:6314` | (a) Remove auto-open (`CMP/login-dialog.ts:3`, `:99-110`) and the native clipboard and platform hints (`CMP/auth-url.ts:2`, `:26-34`). The URL is displayed as text. Opening it is **L:openUrl** (external kind) and copying it is **L:copy**. Both are launcher-confirmed. | +| Prompted values (API key, codes) | pi: plain `Input` (`IM:6295-6297` → `CMP/login-dialog.ts:58-63`), with the submitted value echoed as `> ${value}` (`CMP/login-dialog.ts:80-83`) | **p2 improvement, not pi parity.** For `prompt.type === "secret"`, use a secret-mode input that masks characters and does not echo the submitted value. The value is excluded from history, from the reload snapshot (`P2/src/hosts/tui-worker-runtime.ts:118`) and from status lines. The web already masks (`P2/plugins/login/web.tsx:130`). | +| `/logout`, refresh | `IM:6018`, `IM:6078` | (b) `ProviderLogin.logout` / `refresh`. | +| Ambient auth, subscription warning, Radius offer | `IM:6174`, `IM:5273`, `IM:6389` | (c) Show `ProviderLogin` notices. | + +### 1.9 Sessions + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| `/resume` selector | `CMP/session-selector.ts:694`, `CMP/session-selector-search.ts:39-153`, `IM:5720` | (a) UI with the documented adaptations in section 1.17. Data: `Sessions`, `Search`, `SessionStatus`. | +| `/new` | `IM:6938` | (b) `Sessions.create({cwd, env})`. A separate `/new-context` uses **new** `Chat.reset(handoff?)` (`DUR/harness/types.ts:642`). | +| `/name` | `IM:6680` | (b) `Sessions.rename`. pi distinguishes an explicit session name; p2 rows have only `title` (`P2/plugins/sessions/contract.ts:9-20`). | +| `/session` | `IM:6705` | (b) Section 1.16 provenance. | +| `/tree`, `/fork`, `/clone`, double-Esc action | `CMP/tree-selector.ts:1336`, `CMP/user-message-selector.ts:110`, `IM:5571`, `IM:5512`, `IM:5550`, `IM:3070-3093` | (c) `/tree` and double-Esc; (d) `/fork` and `/clone` via **new** `Sessions.fork` (`DUR/harness/types.ts:663`). | +| `/export`, `/import` | `IM:6532`, `IM:6579` | (c) | +| `/share` | `IM:6623` | p2 sharing overlay (`P2/plugins/sharing/contract.ts:90`). | +| `/compact [instructions]` | `IM:7092` | (b) **new** `Chat.compact` (`DUR/harness/types.ts:647`). | +| Subagent conversations | — | (b) `Subagents`, keyed by (session, conversation) identity; `/agents` and the subagent card in `P2/plugins/subagents/tui.ts`. | + +### 1.10 Slash commands + +Built-ins come from `CA/core/slash-commands.ts:19-44`; hidden ones are handled in `IM:3199-3397`. A command is +registered only in the phase that implements it (section 10). Example: `/copy` appears in P5, not before. + +| Command | p2 route | +| --- | --- | +| `/settings` | Overlay: guest UI prefs (section 4) plus a read-only view of server settings for the owner. | +| `/model`, `/thinking`, `/scoped-models` | 1.7 | +| `/tree`, `/export`, `/import`, `/bug`, `/changelog`, `/trust`, `/debug`, `/arminsayshi`, `/dementedelves` | (c) | +| `/share` | Sharing overlay | +| `/copy` | **L:copy** with trusted confirmation (section 3.2) | +| `/name`, `/session`, `/new`, `/resume`, `/compact` | 1.9 | +| `/hotkeys` | Help overlay generated from the registry (layout of `IM:6821`) | +| `/fork`, `/clone` | (d) | +| `/login`, `/logout` | 1.8 | +| `/reload` | Re-fetch the TUI bundle (`p2.tui.reload`, unbound). The owner additionally gets the dev plugin reload. | +| `/quit` | exit | +| Prompt templates, `/skill:<name>` | (d) Needs a listing object. | +| Extension commands | `TuiCommands` | +| p2-only | `/sessions`, `/chat-mode`, `/artifacts`, `/providers`, `/envs`, `/devices`, `/logs`, `/ports`, `/new-context`, `/agents`, `/files`, `/changes` (section 9) | + +### 1.11 Keybindings + +Defaults: + +- `TUI/keybindings.ts:71-210` (`tui.*`). +- `CA/core/keybindings.ts:25-171` (`app.*`). +- Platform overrides at `CA/core/keybindings.ts:180-199`, using `useWindowsKeybindings` (`CA/core/keybindings.ts:16`). +- Legacy name migration at `CA/core/keybindings.ts:201-261`. + +The config file is `~/.pi/agent/keybindings.json` (`CA/core/keybindings.ts:338`, `packages/coding-agent/docs/keybindings.md`). +Conflicts are only detected among user overrides (`TUI/keybindings.ts:243-262`). + +| pi id | default | p2 | +| --- | --- | --- | +| `app.interrupt` | Esc | same id | +| `app.clear` | Ctrl+C | same id | +| `app.exit` | Ctrl+D on an empty editor | same id, empty editor only | +| `app.suspend` | Ctrl+Z | launcher-owned | +| `app.thinking.cycle` / `.toggle` / `.save` | Shift+Tab / Ctrl+T / Ctrl+S (selector) | same ids | +| `app.model.cycleForward` / `.cycleBackward` / `.select` | Ctrl+P / Shift+Ctrl+P / Ctrl+L | same ids | +| `app.tools.expand` | Ctrl+O | same id | +| `app.editor.external` | Ctrl+G | launcher-owned | +| `app.message.copy` | Ctrl+X | launcher-owned | +| `app.message.followUp` / `.dequeue` | Alt+Enter / Alt+Up | same ids | +| `app.clipboard.pasteImage` | Ctrl+V | launcher-owned | +| `app.session.new/tree/fork/resume` | unbound | same ids, unbound like pi; `p2.sessions` keeps Ctrl+R in the chat context | +| selector-local `app.session.*`, `app.tree.*`, `app.models.*` | various | same ids, scoped to their overlay context | +| — | — | `p2.tui.reload`, unbound (`/reload`); transcript scrolling uses `tui.altScreen.*` | + +### 1.12 Extension UI hooks + +`ExtensionUIContext` is `CA/core/extensions/types.ts:149-300`, implemented at `IM:2633`. Extensions themselves are +(c): p2 plugins are server-selected and bundled. Each hook maps to a plugin contribution (section 7). + +| Hook | Pointer | p2 | +| --- | --- | --- | +| `select`/`confirm`/`input`/`editor` dialogs with `timeout` and `signal` | `CA/core/extensions/types.ts:113-118`, `:151-157`, `:240`; `IM:2690`, `IM:2750`, `IM:2773`, `IM:2831`; `CMP/extension-selector.ts:19`, `CMP/extension-input.ts:18`, `CMP/extension-editor.ts:26` | `ctx.dialog.*` with the same `{timeout, signal}` options and a countdown. `extension-editor` loses its external-editor path (section 11). | +| `notify` | `IM:2955` | `ctx.notify` (also fed by `LogNotices`) | +| `onTerminalInput` | `IM:2592` | (c) No raw interception beyond the command dispatcher. | +| `setStatus` | `IM:2300` | `TuiFooter` | +| `setWorking*`, `setHiddenThinkingLabel` | `IM:2346-2396` | `ctx.working` | +| `setWidget` | `IM:2398`, `IM:2491` | `TuiSlots` `widget-above:*` / `widget-below:*` | +| `setFooter`, `setHeader` | `IM:2524`, `IM:2551` | `TuiSlots` `footer` / `header`; runtime fallbacks (section 7) | +| `setTitle` | `CA/core/extensions/types.ts:211` | through **L:title** | +| `custom()` overlays: sizing/positioning (`overlayOptions`, possibly dynamic), `onHandle` lifecycle | `CA/core/extensions/types.ts:214-228`, `IM:2966` | `ctx.overlay(factory, {overlayOptions, onHandle})` on pi-tui's overlay stack. Handles are disposed on navigation, reload and plugin replacement. | +| editor text access, `addAutocompleteProvider` | `CA/core/extensions/types.ts:231-243` | `ctx.editor`, `TuiCompletions` | +| `setEditorComponent` / `getEditorComponent` | `CA/core/extensions/types.ts:278-281`, `IM:2875` | (c) Decision: one editor. Plugins add widgets. | +| theme access | `CA/core/extensions/types.ts:287-293` | `ctx.theme` read-only; changes only through prefs | +| tools expanded | `CA/core/extensions/types.ts:296-299` | `ctx.view.expandedTools` | +| `registerShortcut` | `IM:2241` | `TuiCommands` | +| `registerMessageRenderer` / `registerEntryRenderer` | `CA/core/extensions/types.ts:1687`, `:1693` | `TuiEntryRenderers` | +| `registerMarkdownTransformer` | `CA/core/extensions/types.ts:1690`, `:1519` | **ordered** `TuiMarkdownTransformers` (section 7) | +| `registerToolRenderer(resolver)` with `next()` | `CA/core/extensions/types.ts:664-667`, `:1696` | **ordered** `TuiToolRendererResolvers` plus the per-name `TuiToolRenderers` (section 7) | + +### 1.13 Queue, abort, retry, notifications, misc + +| Feature | Pointer | p2 route | +| --- | --- | --- | +| Enter steers while busy; Alt+Enter follows up | `IM:3376-3384`, `IM:4487` | (b) Exists (`P2/plugins/chat/tui.ts:87`). | +| Pending area with dequeue hint | `IM:4735` | (a) `buildTranscript(...).queued` (`P2/src/web/transcript.ts:107`, `:569-580`). | +| Alt+Up restores all queued messages | `IM:4519`, `IM:4754` | (b) **new** `Chat.dequeueOwn`. Client-side selection is impossible: `queued` carries no author, and `Chat.dequeue` withdraws an item before checking its author. Other authors' items are moved to their drafts (`P2/plugins/chat/worker.ts:237-242`, `:263-267`). Contract: server-side, atomically select the caller's queued inputs by the server-stamped author, in submission order. Withdraw only those and return their texts in order. Items placed or withdrawn concurrently are skipped and reported as `{restored, skipped}`. | +| Esc while streaming | `IM:3070` | (b) `Chat.stop` (returns the caller's own texts). | +| Drafts withdrawn by others | — | (b) Exists (`P2/plugins/chat/tui.ts:65-79`), keyed per (session, conversation) after P0. | +| Auto-retry | `IM:3744-3770` | (b) | +| Notifications | `IM:4635-4700` | (b) Status lines plus `LogNotices` | +| Copy last message / selection | `IM:6649` | **L:copy** | +| Terminal capabilities: kitty protocol, modifyOtherKeys, images, hyperlinks, true color | `TUI/terminal.ts:222-330`, `TUI/keys.ts:31` setKittyProtocolActive, `TUI/terminal-image.ts:187` setCapabilityOverrides, `:167` getCapabilities | Bug: the launcher's `ProcessTerminal` negotiates kitty, but `BridgeTerminal.kittyProtocolActive` stays false (`P2/src/hosts/tui-worker-runtime.ts:26`). Nobody calls `setKittyProtocolActive` in the worker. Fixed by the `capabilities` message (section 3.5). | +| Exit and terminal restore | `IM:4303`, `IM:4400`, `IM:7102` | Launcher (`P2/src/hosts/tui-client.ts:201-209`), extended in section 3.6. | +| First-time setup, trust selector, image selector, config selector | `CMP/first-time-setup.ts:33`, `CMP/trust-selector.ts:32`, `CMP/show-images-selector.ts:13`, `CMP/config-selector.ts:875` | (c) | + +### 1.14 Settings and interaction details + +From `CMP/settings-selector.ts:60-101` (config), `:482-865` (items) and +`packages/coding-agent/docs/settings.md:13-21`, `:31-44`, `:72-88`, `:94-115`, `:123-143`. +Nested keys are shown as pi spells them. + +| pi setting | Kind in p2 | Notes | +| --- | --- | --- | +| `theme`, `quietStartup`, `editorPaddingX`, `outputPad`, `autocompleteMaxVisible`, `showHardwareCursor`, `hideThinkingBlock`, `doubleEscapeAction` | **G** | `doubleEscapeAction` only supports `none` until `/tree`/`/fork` exist | +| `terminal.clearOnShrink`, `terminal.showTerminalProgress`, `terminal.showImages`, `terminal.imageWidthCells`, `terminal.hyperlinks`, `terminal.images`, `terminal.trueColor` | **G** | Image protocol selection only matters once launcher images exist. Hyperlinks need the URL policy (section 3.4). | +| `markdown.codeBlockIndent`, `markdown.mermaid` | **G** | | +| `tuiMode` | (c) fullscreen only (1.1) | | +| `fullscreenExitOutput` (`transcript` / `resume-hint`) | **G** | `transcript` is (d): the worker would have to hand the launcher a filtered transcript dump on exit. `resume-hint` prints `npm run tui -- <origin>` plus the session. | +| `fullscreenScrollbar`, `fullscreenCopyOnSelect`, `fullscreenWheelScrollLines` | **G** | Copy-on-select is a **L:copy** grant per selection gesture (section 3.2). | +| `externalEditor` | **G**, launcher only | never sent to the worker | +| `images.autoResize`, `images.blockImages` | server | Attachment policy is the server's. Show read-only. | +| `defaultProvider/defaultModel`, `defaultThinkingLevel`, `modelThinkingLevels`, `thinkingBudgets`, `enabledModels` | server | p2 has `defaultModel`/`thinkingLevel` (`P2/src/runtime/settings.ts`); the others are **new** or (c) | +| `steeringMode`, `followUpMode`, `transport`, `httpIdleTimeoutMs`, `websocketConnectTimeoutMs`, `retry.*`, `compaction.*` (incl. `modelOverrides`), `cacheWarming`, `showCacheMissNotices` | server | p2 has a subset (`P2/src/runtime/settings.ts`). Read-only display. | +| `defaultTools`, `codemode.*`, `shellPath`, `shellCommandPrefix`, `npmCommand` | server / env | Tool loadout and shell are server or env configuration (`P2/plugins/envs`). Not TUI settings. | +| warning toggles, `enableInstallTelemetry`, `defaultProjectTrust`, `treeFilterMode`, `branchSummary.*`, `sessionDir`, `httpProxy` | (c) | No equivalent concept in p2. | + +### 1.15 CLI behavior + +pi flags: `CA/cli/args.ts:83-268`, `:300-348`. p2 has only `[url]` and `--owner-link` (`P2/src/hosts/tui-config.ts:92-100`). +The **new** `--join` flag comes from P0. + +| pi flag | p2 | Where | +| --- | --- | --- | +| `--tui-mode` | unsupported (1.1) | — | +| `--verbose` | launcher: shows the expanded header and resource listing | launcher → worker flag | +| `--theme <name>` / `--use-theme`, `--no-themes` | launcher: overrides the **G** theme for this run; `--no-themes` skips custom theme files | launcher | +| initial message args, `@file` args | launcher: the text is posted to the worker as the first draft (not auto-sent). `@file` args are **L:attach** with consent. | launcher | +| `--continue/-c` | launcher remembers the last session per origin; the worker navigates to it | launcher config | +| `--resume/-r` | opens the session selector | worker | +| `--session <id>` | navigates; `--session-id` is an alias | worker | +| `--fork`, `--name` | (d) fork; `--name` renames the session after open (owner) | server | +| `--model`, `--provider`, `--thinking`, `--models` | owner: applied with `Chat.configure` to a new session or the opened one; guests: refused | server | +| `--session-dir`, `--no-session` | unsupported: sessions are server-side | — | +| tool allow/deny (`--tools`, `--no-tools`, `--exclude-tools`, `--no-builtin-tools`), `--system-prompt`, `--append-system-prompt` | unsupported: server configuration | — | +| `--extension`, `--no-extensions`, `--skill`, `--no-skills`, `--prompt-template`, `--no-prompt-templates`, `--no-context-files`, `--no-mcp` | unsupported. Local resource paths would be launcher-owned data; server resources are plugins. | — | +| `--approve`, `--no-approve` (trust) | unsupported | — | +| `--offline` | unsupported (the TUI is a network client) | — | +| `--api-key` | unsupported (credentials are the server's) | — | +| extension-defined flags | unsupported | — | +| `--print/-p`, `--mode json/rpc`, `--export`, `--list-models` | excluded from this guide | — | + +### 1.16 Footer and `/session` data provenance + +| Number | Scope | Source | +| --- | --- | --- | +| Session total tokens/cost | Whole session, including children and tools | `SessionMetrics.usage` (`P2/plugins/status/server.ts:107-125`; caveat already stated in `P2/src/web/CostPanel.tsx:25-26`). Label it "session". | +| Conversation tokens/cost | Viewed conversation | `view.docs["pi.usage"]` (`conversationUsage`, `P2/plugins/status/tui.ts`) or transcript `costs` | +| Cache hit rate | Latest committed response in the viewed conversation | `buildTranscript(...).latestAssistantUsage` → `cacheHitRate` (`P2/src/web/transcript.ts:110-112`) | +| Context tokens | Viewed conversation's active context estimate | `buildTranscript(...).contextTokens` | +| Context window (denominator) | Current model | Owner: `Models.models`. Guests cannot hold `Models` (`P2/plugins/models/server.ts:62-64`). Either add **new** guest-safe current-model metadata (`contextWindow`, `name`, `thinkingLevels` of the conversation's model only) to the `Chat` view state, or show tokens without a percentage. | +| Model and thinking | Viewed conversation | `pi.agent` | + +`/session` shows the same distinctions. + +### 1.17 Session selector adaptations + +pi's selector depends on local full-text session data, which p2 does not have: + +- Fuzzy tokens over id, name, all message text and cwd (`CMP/session-selector-search.ts:26-27`). +- Phrases and `re:` regex (`CMP/session-selector-search.ts:39-153`). +- Threaded sort. +- Tab toggles the scope between the current folder and all sessions (`CMP/session-selector.ts:170`, `:1032-1039`). + +p2's `Search` is literal, returns bounded snippets and at most 41 hits (`P2/plugins/search/contract.ts:14-37`, `:64`, `:90-95`). +Session rows lack an explicit-name flag and branch ancestry (`P2/plugins/sessions/contract.ts:9-20`). + +Adaptations: + +- **Filtering.** Local fuzzy filtering over title, env:cwd and id. Message hits come from `Search.search`, shown in a + separate "Messages" group labeled as literal matches. Phrases and `re:` are unsupported; the hint line says so. +- **Tab scope.** Tab keeps pi's meaning: current env:cwd versus all sessions. +- **Archived.** Archived scope is a separate key: `app.session.toggleArchived`, default Ctrl+A. +- **Sort.** Recent and relevance sorts are supported. Threaded sort groups by subagent parent + (`Subagents`) rather than by forks. +- **Named filter.** pi's named filter becomes a "pinned" filter (`pinned` exists). +- **Delete.** Delete means archive, with restore available in the archived scope. + +Adding regex/phrase search would be a `Search` contract change (d). + +--- + +## 2. Where the port lives in p2 + +| Part | Files | +| --- | --- | +| Launcher (terminal, connection, sign-in, grants, consent UI, modes, output filter) | `P2/src/hosts/tui.ts`, `tui-client.ts`, `tui-config.ts`, `tui-grants.ts`, `tui-actions.ts`, `tui-trusted-ui.ts`, `tui-modes.ts`, `tui-filter.ts`, `tui-local-preferences.ts` | +| Bundle and sandbox | `P2/src/hosts/tui-bundle.ts`, `tui-sandbox.ts`, `tui-bootstrap.mjs`, `tui-worker-runtime.ts` | +| Runtime (screens, overlays, commands, renderer registries, context) | `P2/src/runtime/tui.ts`, `tui-commands.ts`, `tui-renderers.ts`, `tui-context.ts`, `tui-preferences.ts` | +| Copied components | `P2/src/tui/components/` (messages, transcript, tool card, status indicators and editor, selectors, header/footer/status lines), `P2/src/tui/tools/` (tool renderers), `P2/src/tui/theme/` | +| Runtime commands, keybindings, reload snapshot | `P2/src/tui/core-commands.ts`, `keybindings.ts`, `snapshot.ts` | +| Chat surface: transcript `ScrollView` over the dock, pending area, working indicator, sends | `P2/plugins/chat/tui.ts` (`ChatSurface`), `tui-commands.ts` | +| Plugin screens and cards | `P2/plugins/<name>/tui.ts` | + +--- + +## 3. Isolation constraints and launcher grants + +Constraints (`P2/README.md` "TUI isolation", `P2/DEV.md:32-54`, `P2/docs/p2-notes.md` "TUI"): + +- The worker drops fs/net/child/worker/addon/wasi/inspector before evaluation + (`P2/src/hosts/tui-bootstrap.mjs:13-16`). It runs with an empty env and V8 limits + (`P2/src/hosts/tui-sandbox.ts:37-49`). +- All worker output passes `filterTerminalOutput` (`P2/src/hosts/tui-filter.ts:17`). +- The launcher accepts only `output`, `link`, `reload` and `exit` (`P2/src/hosts/tui-client.ts:158-167`). +- `P2/DEV.md:47-49` says any local file, clipboard or browser capability needs "an explicit, launcher-owned user grant + with validated data, not a server-provided filename, URL or simulated prompt". + +Every server and its downloaded code are untrusted relative to the visitor, owners included (`P2/docs/p2-notes.md` +threat model; `P2/README.md` TUI isolation). The rules below follow from that. + +### 3.1 Grant kinds + +A **grant** is the launcher's permission for exactly one action instance. It is bound to: + +- the worker instance (sandbox identity), +- the server origin, +- the target session/conversation that the worker named at request time, and +- an expiry. + +Grants are never inferred from unrelated input. A key press counts only for the action its binding names. + +| Kind | How the grant arises | Actions | +| --- | --- | --- | +| **K: launcher-matched key** | The launcher resolves the guest's own keymap (section 4) on raw input before forwarding. A match for a privileged action id creates a grant for that action only. The key is consumed and not forwarded. | `app.clipboard.pasteImage`, `app.editor.external`, `app.suspend`, `app.message.copy`, `p2.attach` | +| **C: trusted confirmation** | The worker requests an action. The launcher shows its own consent UI (3.3) with launcher-known facts and the exact data. Only a local confirmation creates the grant. | `/copy` and other command-initiated copies, `openUrl`, `savePrefs`, `attach` of a dropped path, `download` | +| **N: no grant** | Sanitized, bounded, rate-limited output with no access to local data | `title`, `progress`, `programStatus` | + +Rules: + +1. **Privileged bindings are launcher-only.** The worker can never change the bindings of privileged action ids, or + the external editor command. A `savePrefs` patch containing them is rejected before confirmation is shown. + Otherwise, a confirmed-looking theme change could also remap Enter to `app.clipboard.pasteImage`. +2. **Unbound by default** if a privileged action would collide with a non-privileged default in the guest keymap + (section 6.2). +3. **The worker may only suspend interception.** It can send `{kind:"keyContext", suspendPrivileged: true}` while a + modal text input owns keys, e.g. transcript search, where Ctrl+G means "next match" (`TUI/keybindings.ts:196-199`). + It can never request interception. A lying worker can only deny the user a privileged action, which falls under + the owner-accepted denial of service. +4. **Copy by key (K)** asks the worker for the text: `request {kind:"copyText", grant}`, which returns the last + assistant text or the selection, bounded to 1 MiB. The launcher writes it. It then shows a launcher-drawn one-line + summary ("Copied 1 234 chars: <first line>") via the trusted UI. That summary does not take input, but it still + suspends worker IO while drawn. +5. **Copy by command (C)**: `/copy`, the copy-URL action in the login dialog, and share-link copy show a preview of + the full text (scrollable, with its length) and need explicit confirmation. Copy-on-select is a K grant from the + selection gesture itself only if the launcher observes the mouse release; otherwise it is C. + +### 3.2 Capability map + +| Capability | Grant | Message / behavior | Notes | +| --- | --- | --- | --- | +| `capabilities` | — | Launcher → worker on `activate` and on change: `{kittyProtocol, modifyOtherKeys, colorDepth, platform, windowsKeys}` | Fixes the kitty bug; Windows defaults follow the guest platform | +| `preferences` | — | Launcher → worker: validated **G** data | Section 4 | +| `terminalColors` | — | Launcher → worker: colors and later scheme changes | The launcher queries the terminal itself | +| `copy` | K or C | 3.1 rules 4-5 | Native clipboard writer copied into the launcher (`CA/utils/clipboard.ts`); OSC 52 written by the launcher only | +| `paste` (right click) | K (mouse gesture observed by the launcher) | The launcher reads text and forwards it as bracketed paste | Text only | +| `pasteImage` | K | The launcher reads the image (`CA/utils/clipboard-image.ts`), then shows trusted confirmation with size and destination, then uploads (3.4), then posts `{kind:"attached", grant, uploadId, name, mime}` | The K grant alone does not upload: the image leaves the machine, so confirmation is required | +| `attach` | K (`p2.attach` opens the launcher file picker) or C (a dropped path) | Trusted picker/confirmation → bounded read (3.3) → upload → `attached` | The only path for local files | +| `externalEditor` | K | `request {kind:"editorText"}` → suspend worker IO → stop the terminal → spawn the **G** command with argv (no shell) on a temp file → restore → `editorResult` bound to the grant | Result dropped if the grant target changed | +| `suspend` | K | The launcher stops the terminal, resets modes (3.6) and raises SIGTSTP; on resume it redraws and posts `resumed` | | +| `title`, `progress`, `programStatus` | N | Sanitized (control characters removed), ≤ 80 chars / enum, ≤ 4 per second | Reset on exit | +| `openUrl` | C | Section 3.5 | | +| `download` | C | Section 3.5 | | +| `savePrefs` | C | `{patch}`. Confirmation shows the actual patch as a diff. Privileged keys and the editor command are rejected. The launcher writes `~/.pi/p2-tui/prefs.json` atomically. | pi's own files are never written | +| `image` | (d) N | Worker → launcher: a bounded PNG; the launcher decodes, re-encodes and emits graphics itself | | +| `terminal` | (d) | Section 3.7 | | + +### 3.3 Trusted consent UI + +The problem: the worker's terminal state (cursor, SGR, scroll region, synchronized output, alt screen) and its +asynchronous output can corrupt or spoof a launcher prompt. If the launcher paused output for one frame only, the +worker could paint over the prompt or read the user's keystrokes. + +Protocol for every C grant, every K action that shows UI, the picker and summaries: + +1. **Suspend worker IO for the whole interaction.** Stop forwarding input to the worker. Buffer worker output + (bounded; on overflow, drop the frames and mark a full redraw). The worker keeps running. +2. **Reset drawing state.** End synchronized output, reset SGR, reset the scroll region, show the cursor, clear the + screen. The prompt is then drawn on a known state. +3. **Show only launcher-known facts and explicit data.** Include: + - the server origin from the launcher's fixed URL; + - the action; + - the file path, size and type (regular files only); + - the destination session ID; the session title may be shown, labeled "server-reported"; + - the full URL, patch or text preview. + + Data from the worker is shown as data, never as instructions. Control characters are stripped. +4. **Consume confirmation keys locally.** Enter or `y` confirms; Escape or `n` cancels. Nothing reaches the worker. +5. **Redraw.** Discard the buffered worker output and send `{kind:"redraw"}`. The worker calls + `tui.requestRender(true)`. +6. **Bind async results.** Every result message carries the grant id. If the worker was replaced (reload swap), or the + worker reports navigation away from the grant's session/conversation, the launcher cancels the operation (abort + the upload, delete temp files) and drops the result. +7. **Read local files safely.** Open with no-follow semantics, `fstat` the opened descriptor, and require a regular + file. Reject devices, FIFOs, sockets and directories. Enforce the size limit before and during reading. Read only + through that descriptor. + +Uploading local data to the connected server is the user's explicit choice under this UI. The confirmation names the +origin because it is an exfiltration channel by design: the server receives what the user confirms. + +### 3.4 Upload and launcher HTTP transport + +All launcher HTTP follows the existing trusted request pattern (`P2/src/hosts/tui-client.ts:128-129`, +`P2/src/hosts/tui-config.ts:37-46` (claimOwner), `:67-71`): + +- **Fixed URLs.** `new URL("/upload", origin)` with `URLSearchParams({session, name})`. No server-supplied path, + query or origin. `session` must match the ID format the worker reported for the grant. `name` is the local basename, + sanitized, at most 255 characters (checked server-side at `P2/plugins/files/server.ts:78-79`). +- **Headers.** `connectionHeaders(origin, credentials)` (`P2/src/hosts/tui-config.ts:34`) supplies the exact + `Origin`, which `/upload` requires (`P2/plugins/files/server.ts:70-71`), and the cookie. +- **Redirects.** `redirect: "error"`, so credentials never follow a redirect. +- **Timeouts.** `AbortSignal` with a timeout (60 s for uploads, 10 s otherwise) and grant cancellation. +- **Size.** At most 25 MiB (`MAX_UPLOAD_BYTES`, `P2/plugins/files/uploads.ts:10`), enforced locally before sending. + Streamed from the verified descriptor. +- **Response.** Read bounded (8 KiB), expecting `{id}`. Errors are shown in the launcher UI, not forwarded verbatim + to the worker. +- **Cookie.** The cookie never appears in URLs, the worker, logs or child processes. + +### 3.5 Opening URLs and downloads + +Browser sign-in and TUI sign-in are separate. The launcher cookie is private to the launcher +(`P2/src/hosts/tui-config.ts:6-35`). Content endpoints authorize the browser independently: + +- `/download` checks the principal from the browser's own cookie and refuses cross-site requests + (`P2/plugins/files/server.ts:89-106`). +- Artifact URLs are authorized the same way. + +Two distinct capabilities follow: + +- **External URL** (`openUrl {kind:"external"}`), e.g. OAuth provider pages from `ProviderLogin.login.notices`. + - `https:` only. + - Shown in full in the trusted confirmation. + - Opened with the OS opener via argv, with no shell and no credentials added. +- **Fixed-origin content** (artifacts, sent files, the web UI for a session). + - **Open in browser.** `openUrl {kind:"origin", path}` with a path from a fixed allowlist (`/s/<id>`, artifact and + download routes). The URL is built from the launcher's origin, so loopback `http:` is allowed when that is the + configured origin. The confirmation says: "The browser needs its own sign-in (owner link or share link)." + The cookie or a token is never put in the URL. + - **Download** (`download {session, id}`, C grant): the launcher GETs `/download` with its own cookie. It uses the + fixed route, `redirect: "error"` and a bounded size, and saves into a launcher-chosen folder (G pref, default + `~/Downloads/p2/`) under a sanitized name, with no overwrite. +- **Hyperlinks** (OSC 8) stay stripped by the filter. A later filter rule may pass OSC 8 only for `https:` URLs whose + visible text equals the URL. Raw OSC 8 is never allowed blindly. + +### 3.6 Terminal modes owned by the launcher + +`TuiAltScreen` writes these sequences (`TUI/tui-alt-screen.ts:63-75`, `:372-398`): + +- autowrap off/on (`?7`); +- mouse modes 1000/1002/1006, optionally 1003; +- focus reporting (1004); +- synchronized output; +- OSC 133 markers. + +- **Allowed modes.** The filter allows `?7` and, if the guest's mouse policy permits, `?1000`, `?1002`, `?1006`, + `?1004`. `?1003` (all-motion) is allowed only with `fullscreenWheelScrollLines`/hover features enabled by the guest. +- **Tracking.** The launcher records every mode the filtered output enabled. +- **Cleanup.** On exit, suspend, worker crash, worker swap and launcher error, the launcher writes the inverse of every + enabled mode plus a fixed reset: `?1000l ?1002l ?1003l ?1006l ?1004l ?7h ?2004l ?25h ?2026l`, SGR reset, scroll + region reset, `?1049l`. A swap re-enters cleanly. Kitty keyboard state is the launcher's own + (`ProcessTerminal.stop`). +- **OSC 133.** The markers stay internal. `TuiAltScreen` strips them from its screen lines (`TUI/tui-alt-screen.ts:411`, + `:1690`), and the filter strips all OSC regardless. + +### 3.6a As implemented (P0b) + +- **Grants** (`P2/src/hosts/tui-grants.ts`). `Grants.create(action, "K" | "C", worker, target)` binds a random id to + the sandbox instance, the launcher's origin, the target the worker last reported and a 10-minute expiry, with an + `AbortSignal`. `valid(id, worker, action?)` checks all of that. A swap or crash cancels the worker's grants, a + `{kind:"navigation", session, conversation}` report cancels grants for another target, close cancels everything. + Results of a cancelled grant are dropped (`{ok:false, error:"cancelled"}`). +- **Actions** (`LauncherAction`): `K` actions name a privileged keybinding id and run with a context whose + `ask(kind, payload)` asks the granting worker for data (`grantRequest`, answered from `runtime.launcherAnswers`); + `C` actions validate the payload in `consent(payload)` (undefined denies without UI) and `run` only after + confirmation. One interaction at a time; a request while busy is denied. Local error text stays in the launcher + (a launcher notice); the worker learns only `failed`. P0b ships the framework; P5 registers the actions. +- **Worker API.** `ctx.grant(action, payload)` → `{kind:"request", id, action, payload}` → `{kind:"response", id, + ok, value | error}`. The worker runtime reports `navigation` and `keyContext` after every input and render. +- **Privileged matching** (`PrivilegedKeys`). Built from the visitor's local keybindings only (pi files and p2 + prefs, never server defaults or the worker) and the guest platform, through `resolvePrivilegedKeys`; only ids with + a registered K action are intercepted. A match is consumed before forwarding (key releases too, without firing). +- **keyContext** (rule 3). The worker sends `{kind:"keyContext", suspendPrivileged}` whenever pi-tui's focus is not + the runtime itself (transcript search, a dialog's text input). The launcher stores only that boolean, resets it on + swap and ignores everything else in the message. +- **Trusted UI** (`P2/src/hosts/tui-trusted-ui.ts`). `confirm(request)` and `notice(text)`. While active the client + discards worker output (and title/progress messages) and gives every key to the UI. Drawing starts with + `?2026l`, SGR reset, scroll-region reset, cursor shown and a clear, then an inverse bar "p2 launcher · on this + computer · not the server's screen", `Server:` (the launcher's URL), `Action:`, facts (the target session is added + by the launcher), and the data preview, scrollable, with control characters made visible (␛, ␇, …). Only Enter or + `y` confirms; Escape, `n` and Ctrl+C cancel; other keys scroll or do nothing. On close it re-applies the worker's + tracked modes and posts `{kind:"redraw"}`; the worker calls `tui.requestRender(true)`. +- **Modes** (`P2/src/hosts/tui-modes.ts`, `tui-filter.ts`). The filter admits `?7` (the autowrap fix) and the mouse + modes pi's alt screen uses, by the visitor's launcher-only `prefs.json` key `mouse`: `"on"` (1000/1002/1006/1004, + button motion), `"all"` (also 1003), `"off"`. The default is pi's: `"on"` inside tmux/screen/zellij, else `"all"` + (`defaultMouse`, the same environment test pi-tui makes). `TerminalModes` records every private mode the worker set; `cleanup()` writes their inverses + and the fixed reset of 3.6 (also `?1049l`) on exit, crash and before a swapped worker activates; `restore()` puts + them back after launcher UI. +- **Mouse** (owner request after P5). The worker runs pi-tui's `TuiAltScreen` as is with the mouse captured unless the + policy is `"off"` (`input.mouse` in the sandbox's start data): wheel scrolling, selection, scrollbar dragging and + copy on select are pi-tui's code. pi's `fullscreenWheelScrollLines` (default `"auto"`) and `fullscreenCopyOnSelect` + (default true) are applied from the visitor's settings. The only p2 part is pi-tui's clipboard write: its + `copySelection` hook requests the `copy.selection` launcher action (instead of the OSC 52 the filter strips), and + the launcher accepts it once, within two seconds after it saw, in the raw input, the left-button release ending a + drag or a multi-click (`P2/src/hosts/tui-mouse.ts`, `SelectionGesture`). No consent screen: the launcher observed + the gesture; pi-tui flashes "Copied!" or the failure from the answer. A request without that gesture is denied. +- **N actions.** `BridgeTerminal.setTitle/setProgress/setProgramStatus` post `title`, `progress` and `programStatus`. + The launcher strips controls, bounds titles to 80 characters, applies at most four per second per kind (the latest + throttled value follows), gates progress on `terminal.showTerminalProgress`, and on exit clears the title and + progress and sends program status `clear`. The worker titles the window `π p2 · <host>` and reports progress and + working/idle from the chat's `busy` context. +- **Tests:** `P2/test/p2/tui-grants.test.ts` (modes, trusted UI keys and visible data, grant binding, privileged + matching, an isolation run with a scriptable fake worker: K only on its key and not while suspended, C denied by + Escape/`n`, worker output and input suspended during consent, confirmation runs exactly once, navigation during + consent cancels, unknown actions denied, title sanitizing and rate limit, mode cleanup on swap and crash; kitty + decoding in the real worker both ways; search suspending Ctrl+G in the real worker). Suspend resets are tested with + `app.suspend` in P5. + +### 3.6b As implemented (P5): launcher actions + +`P2/src/hosts/tui-actions.ts` (`launcherActions`), wired by `P2/src/hosts/tui.ts`. Workers call C actions with +`ctx.grant(name, payload)` and get `{ok: true, value}` or `{ok: false, error}` with `error` one of `denied` (invalid +payload, unknown action, or cancelled on the consent screen), `busy`, `cancelled` (the grant ended: navigation, swap) +or `failed` (the launcher shows its own error text; the worker never sees it). Session-bound payloads must name the +session the worker last reported (`navigation`). + +| Name | Kind | Payload | Launcher behavior | +| --- | --- | --- | --- | +| `copy` | C | `{text, label?}`; text non-empty, ≤ 1 MiB; `label` is shown as "(server-reported)" | Shows the full text (scrollable) and its length; copies on confirmation, then a summary notice. `/copy` (chat) uses it with the last answer. | +| `copy.key` | K `app.message.copy` (Ctrl+X) | — | Asks the worker `copyText` → `{text}` (≤ 1 MiB; chat answers the last answer); copies and shows "Copied N characters: <first line>". | +| `openUrl` | C | `{kind: "external", url}` (https, no credentials, ≤ 2048) or `{kind: "origin", path}` | Origin paths: `/s/<uuid>`, `/artifact?session&name&version[&download]`, `/download?session&id`, built on the launcher's origin; content paths must be the bound session. The screen shows the full URL and, for origin paths, "the browser needs its own sign-in". Opens with the platform opener by argv, no shell, nothing appended. | +| `download` | C | `{kind: "file", session, id, name?}` or `{kind: "artifact", session, name, version}` | `GET /download` or `/artifact?download=1` on the fixed origin with the launcher's cookie, `redirect: "error"`, 60 s, ≤ 25 MiB (declared and streamed); saves into the download folder (launcher-only `prefs.json` `downloads`, default `~/Downloads/p2`) under a sanitized name with `O_EXCL` (`name (1).ext`, never overwrites); partial files are removed. The local path is shown in a notice, never returned (`{saved: true}`). | +| `editor.external` | K `app.editor.external` (Ctrl+G) | — | Asks `editorText` → `{text}` (null when there is no editable editor), writes a 0600 temp file, lends the terminal (worker IO suspended, modes undone, terminal stopped), runs the command by argv, restores, and sends `editorResult` `{text}` through the grant, so it is dropped after navigation or a swap. The command is launcher-only: `prefs.json` `externalEditor`, pi's `externalEditor`, `$VISUAL`, `$EDITOR`, then `nano`/`notepad`; split into argv without a shell (quotes group). | +| `suspend` | K `app.suspend` (Ctrl+Z) | — | Lends the terminal and raises SIGTSTP (keep-alive, SIGINT ignored while stopped); on SIGCONT restores modes and the worker redraws. Not on Windows. | +| `devCommand` | C, owner | `{action: "reload"}` (nothing else) | `POST /__p2_dev` with the launcher's sign-in and exact Origin; returns `{message}` from the server. The server decides whether it is a dev server. | +| `signOut` | C, owner | `{all: boolean}` | `POST /owner/logout` or `/owner/logout-all`. Denied for guests, and `all: false` on loopback (the device owner has no cookie). The server then revokes the connection, so the launcher shows "Signed out on this device." and the owner-link prompt (P0a), forgetting the cookie. | + +Worker side: the chat surface answers `copyText`, `editorText` and `editorResult` (`registerLauncherAnswers`, +`P2/plugins/chat/tui-commands.ts`) and registers `/copy` (`p2.chat.copy`). `copy.selection` (kind `gesture`, +payload `{text}`) is pi-tui's copy on select (3.6a, Mouse); `copy.key` copies the last answer. **Deviations:** +`devCommand` is not restricted to loopback in the launcher (the server's route already requires the +owner and a dev server, and the consent screen confirms). Tests: `P2/test/p2/tui-actions.test.ts`. + +### 3.7 Terminals (shell panels) + +The worker cannot open the terminals byte channel (a separate authenticated upgrade, `Terminals.channel`, +`P2/plugins/terminals/contract.ts:17-46`). Raw output must never bypass the filter, the owner's included. + +- **(d) Filtered terminal.** + - The launcher opens the fixed-origin byte channel with the token the worker obtained from `Terminals.channel`. + The route is fixed; the token is opaque and bounded. + - The launcher relays the bytes to the worker as opaque frames, the same way it relays chord frames. + - The worker runs a headless VT emulator and renders cells as ordinary filtered output. + - Owner only, because `Terminals` is owner-only. +- **(d, optional) "Trust this server's terminal like SSH".** + - A separate launcher mode, confirmed in the trusted UI and stored per origin in the launcher config, revocable. + - Only in this mode does the launcher pass raw pty bytes. Even then it applies its exit cleanup. + - Never enabled by a server-side owner variant or a worker request. + +--- + +## 4. Guest admission and guest-local preferences + +### 4.1 Guest admission (P0) + +- **Redemption.** `--join <link>`, or a share link pasted at the prompt, where the prompt accepts either an owner link + or `https://host/join/<token>` (`P2/plugins/sharing/server.ts:46`). + - The launcher checks the link's origin against the URL argument (or takes the URL from the link when no URL is + given). + - It prompts for a display name. That name is unverified; the server stamps it on the guest. + - It POSTs `/join` with `{token, name}` (`P2/plugins/sharing/server.ts:297-336`), using `connectionHeaders` (exact + Origin is required, `P2/plugins/sharing/server.ts:304`), `redirect: "error"` and a 2 KiB body. + - It stores the returned `Set-Cookie` value per origin, with role `guest` and the session ID. +- **Check on start.** Owner cookies use `/owner/check`. Guest cookies are checked by fetching `/__p2_tui`, which is + admitted for guests, and then the `Access` object yields the grant. A 401/403 means the share was revoked or reset, + or the session was archived: delete the stored cookie and prompt for a new link. +- **Revocation while running.** If `Access` becomes gone or forbidden, show a launcher-drawn notice ("Access to this + session ended") and exit or prompt. No guest preferences or presence are sent before admission succeeds. +- **Loopback.** Direct loopback is always the device owner (`P2/src/hosts/tui-config.ts:60-62`), so guest admission + applies to remote origins and proxied local origins. +- **Presence.** After admission the worker holds `GuestPresence` and shows "Guest · name · view/full" in the header. + +**As implemented (P0a).** + +- **CLI and prompt** (`P2/src/hosts/tui-config.ts`). `--join <link>` takes the origin from the link when no URL is + given, otherwise the link must match it (`joinLinkToken`: `/join/<43-char token>`, no query/fragment/credentials). + `--join` and `--owner-link` exclude each other. The sign-in prompt accepts an owner link or a share link; a share + link asks for a name (one line, control characters stripped, at most 40 characters, asked again when empty). +- **Redemption** (`redeemJoin`): `POST /join` with `connectionHeaders` (exact `Origin`), `redirect: "error"`, a 10 s + timeout and a 2 KiB response cap. The stored guest cookie of that origin is sent along, so rejoining keeps the + identity (the server's rule). Server error text is reduced to one plain line. The invitation is never stored. +- **Storage.** `config.json` keeps a guest beside the owner cookie per origin: `{cookie?, guest?: {cookie, session}}`. +- **Start.** Remote origins: a valid owner cookie (`/owner/check`), else a valid guest cookie (`HEAD /__p2_tui`; + 401/403 forgets it and prompts again), else the prompt. Loopback is the device owner; a stored guest cookie never + downgrades it. **Deviation:** an explicit `--join` on loopback joins as a guest for that run (useful for trying a + guest view locally). Note that on loopback the server itself treats a cookie-less request as the device owner, so a + revoked loopback guest silently becomes the owner on reconnect; revocation detection therefore applies to remote + origins. +- **Revocation while running** (`P2/src/hosts/tui-client.ts`, `P2/src/hosts/tui.ts`). A WebSocket upgrade refused + with 401/403, or a 401/403 from the bundle poll, is confirmed by the launcher's own `HEAD /__p2_tui` (the web page + asks the same question with `HEAD /`). The client closes and, on the restored terminal, the launcher prints the web + page's words by the cookie kind it stored, never the worker's role: a guest cookie gives "This share has ended." and + "Paste a new share link to join again"; an owner cookie gives "Signed out on this device." and an owner-link prompt. + It forgets only that cookie; a valid link for the same origin signs in again and starts a fresh client, Enter quits + (`resumeAfterEnded`). The worker cannot trigger this beyond ending its own session. +- **Worker.** The header shows `guest · <name> (unverified) · view only|full access` from `Access`; the new + `P2/plugins/sharing/tui.ts` holds `GuestPresence` from a slot surface (holding it during facet load would block the + worker's startup, because the link activates only after `ready`), so the owner sees the TUI guest as connected. +- **Tests:** `P2/test/p2/tui-guest.test.ts` (argument rules, mixed prompt, restart reuse, ended shares re-prompt, + rejoin identity, loopback owner, and an end-to-end guest through a Host/Origin-rewriting tunnel: header, presence, + no cookie in the worker, revocation on link reset). + +### 4.2 Preference split + +Rule: preferences about how this person operates a terminal live with the person (the launcher). Preferences about +what the agent does live with the server or session. + +| Preference | Owner of the data | Source | +| --- | --- | --- | +| Keybindings | Guest | `~/.pi/agent/keybindings.json` (migrated with a copy of `CA/core/keybindings.ts:201-261`), then `~/.pi/p2-tui/prefs.json` | +| Theme, custom themes, light/dark pair | Guest | pi `settings.json` `theme` + `~/.pi/agent/themes/*.json`, then p2 prefs | +| Display/editor keys from section 1.14 marked **G** | Guest | pi `settings.json` (nested keys as in 1.14), then p2 prefs | +| External editor | Guest, launcher only | `externalEditor`, `$VISUAL`, `$EDITOR`; never sent | +| Mouse policy, download folder | Guest, launcher only | p2 prefs | +| Model, thinking, agent behavior | Session / server | `Chat.configure`, `<data>/settings.json` | +| Credentials | Server | never leave the server | +| Last session per origin | Guest launcher | `~/.pi/p2-tui/config.json` | + +Flow: + +1. The launcher validates the files with copied schemas: `CA/core/key-id-schema.ts` and `THM/theme-schema.ts:52`. + Input is size-capped (256 KiB) and only whitelisted keys pass. +2. It posts `{kind:"preferences", ...}` after admission and before `activate`, and again on file change (the launcher + watches the files). +3. Writes go only through `savePrefs` (C grant, 3.2) into p2's own file. pi's files are only read. +4. Server `TuiPreferences` becomes optional server defaults. Precedence: built-in < server defaults < pi files < + p2 prefs. Privileged-action bindings never come from the server. + +Server code can read and relay whatever preferences reach the worker. Send only the whitelisted subset; never send +env, paths or commands. + +### 4.3 As implemented (P1) + +- **Launcher** (`P2/src/hosts/tui-local-preferences.ts`): reads `$PI_CODING_AGENT_DIR` or `~/.pi/agent` + (`keybindings.json`, `settings.json`, `themes/*.json`) and `~/.pi/p2-tui/prefs.json` (`P2_TUI_CONFIG_DIR`) through + bounded descriptor reads (regular files, 256 KiB each; at most 64 themes, 1 MiB in total). Keybindings pass the copied + key-id pattern and legacy-name migration (`P2/src/tui/keybindings.ts`); settings pass the whitelist + `TuiSettingsSchema` (`P2/src/runtime/tui-preferences.ts`, the **G** keys of 1.14, nested as pi spells them); + themes pass the copied `validateThemeJson`. Built-in/system names are reserved, as in pi. The launcher watches the + three directories (`fs.watch`, re-armed on change, 100 ms debounce) and re-posts only changed preferences. +- **`prefs.json`**: `{"keybindings": {...}, "theme": "light/dark", "settings": {...}}`, the same shapes as pi's files, + overriding them. Launcher-only keys will join it in P5/P9 and are never posted. +- **Message**: `{kind:"preferences", preferences:{keybindings, theme?, settings, themes, problems}}`. `problems` are + display-only diagnostics (labels like `settings.json: invalid terminal.imageWidthCells`), without paths or contents; + `/hotkeys` lists them. The worker re-checks the shape (`parseLocalPreferences`). +- **CLI**: `--theme <setting>` (this run) and `--no-themes` (skip custom theme files), as pi. +- **Server defaults**: `<P2_DATA_DIR>/tui.json` is now `{"keybindings": {...}, "theme": "dark", "settings": {...}}` + (`TuiPreferences` version 2). The old `{"theme": {"accent": 35}}` SGR object is ignored. Privileged ids are dropped + on the server and again in the worker (`effectivePreferences`). +- **Terminal facts**: the launcher posts `{kind:"capabilities", trueColor, platform, windowsKeys, kittyProtocol}` + (before `activate`, and again when kitty negotiation finishes) and `{kind:"terminalColors", terminal:{colors, + scheme}}`. It queries OSC 10/11/4 + DA1 itself (100 ms timeout, late replies still apply), enables mode 2031 + light/dark reports, requeries on a report, consumes the replies (they are never forwarded) and disables 2031 on exit. + COLORFGBG is evaluated by the launcher as the initial scheme. **Deviation:** this is the subset of P0b's + `capabilities` that themes and platform key defaults need; P0b adds `modifyOtherKeys` and the rest. + +--- + +## 5. Themes: from one global to per-instance + +### 5.1 How coding-agent does it + +- `theme` is a `Proxy` over `globalThis[Symbol.for(...)]` (`THM/theme.ts:648-664`), set by `initTheme`, + `setTheme` and `setThemeInstance` (`THM/theme.ts:682`, `:698`, `:721`). +- Module state lives beside it: + - `currentThemeName` (`THM/theme.ts:666`) and `registeredThemes` (`THM/theme.ts:670`); + - one `onThemeChange` callback slot (`THM/theme.ts:730`); + - terminal colors and scheme (`THM/theme.ts:133-157`). +- `Theme` itself reads module terminal state: `appearance` and `colors` (`THM/theme.ts:249-260`). +- `Theme`'s text attributes use Chalk (`THM/theme.ts:314-331`), whose color level is a process global. +- 46 files import the proxy. Among them: every message and tool component, `CMP/keybinding-hints.ts`, + `CMP/diff.ts`, `REN/bash.ts:12`, `CMP/footer.ts`, and the derived theme factories (`THM/theme.ts:1022-1086`). + These factories return closures that read the global. + +Problems: + +- **One theme per process.** Theme previews, two runtimes in tests, split views and server-side rendering cannot + differ. +- **Hidden reads and baked strings.** Many components bake colors into strings at update time, not at render time. + For example, `ToolExecutionComponent.updateDisplay` (`CMP/tool-execution.ts:270-276`) and its fallbacks + (`CMP/tool-execution.ts:142-159`), plus the header. A swap needs explicit rebuilds, not just `invalidate()`. +- **Process coupling.** Global terminal colors and the file watcher tie a theme to one terminal and filesystem. + +p2 runs one worker per viewer, so a global would be per-viewer today. Per-instance is still needed for previews, tests +and keeping module state out of hot-reloaded facets. + +### 5.2 pi-tui assumptions + +pi-tui components take explicit theme objects. Its other globals are per-terminal facts, acceptable per worker and +set from launcher messages: + +- `getKeybindings()` (`TUI/keybindings.ts:315`), used by `Editor`, `Input`, `SelectList`, `SettingsList`, + `CancellableLoader` and `TuiAltScreen`; +- `getCapabilities()` (`TUI/terminal-image.ts:167`) and `setCapabilityOverrides` (`TUI/terminal-image.ts:187`); +- the kitty flag (`TUI/keys.ts:25`, `:31`). + +### 5.3 Design + +- **Copy.** `P2/src/tui/theme/` holds: + - `Theme` from `THM/theme.ts:185-373`, without fs, watchers, `globalThis` or registries; + - `THM/theme-tokens.ts`, `THM/theme-schema.ts` and `THM/system-theme.ts`; + - the two built-in JSON files. +- **Construction.** `new Theme({json, terminalColors, colorDepth})`. Terminal colors and appearance are per instance; + `appearance`/`colors` read the instance fields, not module state. +- **No Chalk.** Bold, italic and similar attributes emit SGR directly at the instance's color depth. +- **`ThemeRef`.** Owned by `TuiRuntime`: `{current, version, subscribe}`. + - Derived pi-tui themes (`markdown()`, `selectList()`, `editor()`, `settingsList()`) are created once per `ThemeRef`. + - Each callback dereferences `ref.current` on every call: `heading: (t) => ref.current.fg("mdHeading", t)`. + - So a swap never leaves stale derived objects. +- **Components.** + - Every component gets `ctx` (`{theme: ThemeRef, keys, hints, ...}`). + - Render-time colors use `ctx.theme.current`. + - Components that bake strings implement `themeChanged()`, which rebuilds cached content (re-running `updateDisplay` + for tool cards, the header text, bash output). The runtime calls it, then `tui.invalidate()` and + `requestRender(true)`. + - Caches of rendered lines are keyed by `ThemeRef.version`. +- **Adapting copies.** + - The `theme` import becomes `ctx`. + - `keyHint(...)` becomes `ctx.hints.key(...)`. + - Tool renderers take `ctx` through the render context; remove their global imports. + - `ThemedText` (`CMP/themed-text.ts:11`) takes `(theme) => string`. +- **Highlighting.** `ctx.theme.current.highlight(code, lang)` via lowlight. Scope classes map to `syntax*` tokens. +- **Preview.** The theme picker builds a second `ThemeRef` for its preview pane only. +- **Tests (P1 acceptance).** + - Two runtimes in one process with different themes render different SGR. + - Swapping a theme rebuilds an already rendered tool card. + - The preview does not change the live transcript. + - Terminal-color updates affect only the instance they were given to. + +**As implemented (P1).** For P0c/P2 to build against: + +```ts +// P2/src/tui/theme/theme.ts (copied tokens/schema/system theme beside it; built-ins in builtin-themes.ts) +class Theme { + readonly name?: string; readonly mode: TerminalColorMode; readonly terminal: TerminalState; + get appearance(): "dark" | "light"; get colors(): Readonly<Record<ThemeToken, Color>>; + fg(token: ThemeColor, text): string; bg(token: ThemeBg, text): string; style(text, ThemeStyle): string; + bold/italic/underline/inverse/strikethrough(text): string; // plain SGR, no Chalk + getFgAnsi/getBgAnsi; getThinkingBorderColor(level); getBashModeBorderColor(); + highlight(code, lang?): string[]; // lowlight `common`, syntax* tokens, per line + static language(hint): string | undefined; // fence info / file path → registered language +} +type TerminalState = { colors: TerminalColors; scheme?: "dark" | "light"; pending?: boolean }; +createTheme(json, mode, terminal?): Theme; createSystemTheme(mode, terminal?): Theme; +resolveTheme({ setting, themes, mode, terminal }): { theme; error? } // pairs, built-ins, custom data, system fallback + +// P2/src/tui/theme/ref.ts +class ThemeRef { get current(): Theme; version: number; set(theme); subscribe(fn): () => void; + readonly markdown: MarkdownTheme; selectList; editor; settingsList } // created once, deref `current` per call +``` + +`TuiRuntime.appearance` (`P2/src/runtime/tui-appearance.ts`) merges server defaults and the launcher's preferences, +holds the terminal state and capabilities, and swaps `runtime.theme` (a `ThemeRef`) only when the selected theme, its +JSON, the terminal state or the color mode changed; it then invalidates and fully re-renders. The color mode is +`settings.terminal.trueColor ?? capabilities.trueColor`. The legacy global helpers (`P2/src/tui/theme.ts`) are gone +(P2): every surface reads its runtime's `theme`; `safeText`/`lineText` live in `P2/src/tui/render-context.ts`. + +### 5.4 Contract between rendering (P0c/P2) and themes/commands (P0d/P1) + +P2 and P1/P0d land in parallel. P1's `Theme` and `ThemeRef` (`P2/src/tui/theme/theme.ts`, `ref.ts`) arrived first, so +rendering consumes them directly; there is no interim fallback theme. + +- **`TuiRuntime.theme: ThemeRef`.** Every surface and copied component reads colors through it (`ref.current.fg(...)`, + `ref.markdown`, `ref.selectList`, `ref.editor`). P1 replaces the instance or calls `ref.set(theme)` on preference + changes. +- **Baked strings.** Components that bake colored strings (tool cards, user/assistant messages, compaction, header, + footer) remember `ref.version` and rebuild in `render()` when it changed, so a theme swap needs no per-component hook; + the runtime only invalidates and re-renders. +- **`RenderContext`** (`P2/src/tui/render-context.ts`, P2) is the structural subset rendering needs: + `{ theme: ThemeRef; keyText(id): string; requestRender(); cwd; ... }`. P0d's `TuiContext` is a superset; + `keyText` is `ctx.hints.key`. Copied components take it instead of the global `theme`/`keyHint`. +- **Runtime split.** P0c/P2 own `TuiRuntime`'s identity (`session`, `conversation`, `identityKey`), drafts, + submissions, scroll anchors, the layout root (header, body, dock, footer), status lines and the reload snapshot. + P0d owns input dispatch (`handleInput`, commands, contexts, hints). Both edit `P2/src/runtime/tui.ts` with targeted + edits only. +- **Surfaces and layout.** `TuiSurface.layout?(): { body, dock? }` gives the runtime a scrolling body (the chat's + transcript `ScrollView`) and a docked part (pending area, editor). The runtime's root `VStack` is body, then dock + (status lines, the surface dock, footer). Slots `header`/`footer` override the built-in header/footer; the status + plugin registers `footer`. Transcript scrolling uses pi-tui's `tui.altScreen.*` keys (the `p2.pageUp/pageDown/bottom` + overrides are gone). +- **Chat actions for commands.** P2 rewrites `P2/plugins/chat/tui.ts`; `ChatSurface` exposes `send(mode)`, `stop()`, + `dequeue()`, `clear()` (Ctrl+C), `editor` and `contexts()` (`editorFocused`, `editorEmpty`, `busy`, `participant`) + for P0d's command table, which registers the keys. + +--- + +## 6. Keybindings and commands + +### 6.1 coding-agent's wiring + +- Static ids (`TUI/keybindings.ts:71`, `CA/core/keybindings.ts:25`) typed by interface merging + (`CA/core/keybindings.ts:176`). +- Ad hoc wiring: + - `CustomEditor.onAction` (`CMP/custom-editor.ts:84`) and the Esc/Ctrl+D special cases + (`CMP/custom-editor.ts:101-125`); + - `IM:3067-3135`; + - per-selector `kb.matches`; + - raw-key extension shortcuts (`IM:2281-2296`). +- No context model; conflicts are detected only among user overrides. +- `/hotkeys` is hand-written (`IM:6821`); slash commands are an if-chain (`IM:3199-3397`). + +### 6.2 End-to-end precedence contract + +Each input chunk passes these stages in order. The first stage that consumes it wins. + +1. **Launcher (trusted).** Privileged action ids from the guest keymap (3.1), unless the worker has suspended + interception for the current modal context. Launcher-drawn UI, when active, consumes everything (3.3). +2. **pi-tui global and alt-screen handling.** Debug, mouse, search when open, then viewport keys unless an overlay + defers (`TUI/tui-alt-screen.ts:729-793`). p2 cannot reorder this. It configures it instead: transcript scroll and + search ids are pi's `tui.altScreen.*` keys, and p2 commands must not claim them in contexts where the alt screen + handles them. The dispatcher reports such claims as conflicts. +3. **Overlay stack.** The focused overlay's commands, then the overlay component. +4. **Commands by context.** Matched against the current context set; more `when` terms means more specific. +5. **Focused component** (usually the editor) with `tui.editor.*` / `tui.input.*`. + +Collision handling: + +- **Privileged vs anything.** At keymap build, the launcher checks each privileged key against all non-privileged + bindings it knows: `tui.*`, `app.*` and the guest's p2 prefs. On a collision, the privileged action keeps the key + only if pi has the same default. Example: pi's Ctrl+G is both `app.editor.external` and + `tui.altScreen.searchNext`; pi resolves it by mode, and p2 by the worker's `suspendPrivileged` during search. + Otherwise the privileged action loses the key and `/hotkeys` reports it. +- **Commands vs commands.** Same key and same context specificity: the user binding wins over defaults. Remaining ties + resolve to the first registered command and are reported. +- **Plugin defaults never claim privileged keys**; the dispatcher drops such defaults. + +### 6.3 Command registry + +```ts +// P2/src/runtime/tui-commands.ts +export type TuiCommand = { + title: string; + category?: string; + keys?: KeyId | KeyId[]; // defaults; pi defaults for pi ids + when?: string; // "chat && editorEmpty && !overlay" + slash?: { name: string; argumentHint?: string; complete?(prefix: string, ctx: TuiContext): Awaitable<AutocompleteItem[] | null> }; + run(ctx: TuiContext, args?: string): void | Promise<void>; +}; +export const TuiCommands = defineRegistry<TuiCommand>("p2.tui-commands", { conflict: "error" }); // key = command id +``` + +- **Ids.** pi ids where the semantics match (`app.model.select`), otherwise `<plugin>.<verb>`. +- **Contexts.** `chat`, `sessions`, `overlay:<id>`, `editorFocused`, `editorEmpty`, `busy`, `compacting`, `searching`, + `owner`, `guest`, `participant`. `when` is a small boolean grammar. +- **Keymap rebuild.** pi-tui's `KeybindingsManager` has fixed definitions and only `setUserBindings` + (`TUI/keybindings.ts:231-305`). When the `TuiCommands` table or prefs change, the runtime builds a **new** manager + from `TUI_KEYBINDINGS`, the copied `app.*` table, plugin defaults and user bindings, then calls `setKeybindings(new)` + (`TUI/keybindings.ts:311`). pi-tui components read `getKeybindings()` per input, so they pick it up. +- **Dynamic ids.** Plugin ids are not in the `Keybinding` union. All matching goes through + `ctx.keys.matches(data, id: string)`, a helper that casts once internally. +- **Slash names.** Validated separately from command ids. The registry's `conflict: "error"` catches only duplicate + ids. The runtime builds a slash-name index: a duplicate name disables the later command's slash form and is + reported in `/hotkeys`. The command id stays usable. +- **Discoverability.** + - Palette `p2.palette` (Ctrl+Shift+P and F1; unbound in pi). + - `/hotkeys`, generated by category and context, showing conflicts, launcher-owned keys and config paths. + - `/` autocomplete; hints via `ctx.hints.key(id)`, a copy of `CMP/keybinding-hints.ts:34-48`. +- **Compatibility.** pi `keybindings.json` (with legacy migration); unknown ids are kept and reported; guest-platform + defaults. + +This is better than coding-agent's approach because one registry drives keys, palette, slash commands and help, with +contexts, plugin contributions, defined precedence and visible conflicts. It also matches the root `AGENTS.md` rule +against hardcoded key checks. + +### 6.4 As implemented (P0d) + +- **Files.** `P2/src/runtime/tui-commands.ts` (`TuiCommand`, `TuiCommands`, the `when` parser, the dispatcher and the + slash completion provider), `P2/src/runtime/tui-context.ts` (`TuiContext`, a superset of P2's `RenderContext`), + `P2/src/tui/keybindings.ts` (pi's `app.*` table per guest platform, the `KEYBINDINGS` overrides, legacy migration, + key-id validation, privileged resolution), `P2/src/tui/core-commands.ts` (palette, `/hotkeys`, exit, reload, view + toggles). +- **`TuiCommand`** is the 6.3 shape plus `palette?: false` (hidden from the palette). `tuiFacet(name, factory, + commands)` registers a slot and its commands. Ids from pi's table always use pi's defaults (`keys` is ignored); + `tui.*` and launcher ids are rejected. +- **`TuiContext`** (`runtime.ctx`): `host, theme, keys {manager, matches, getKeys}, hints {keyText, keyDisplayText, + key, raw}, keyText, requestRender, settings, role, identity, navigate, view {expandedTools, showThinking}, commands + {run, runSlash, autocomplete, palette, hotkeys, list, contexts, report}, surface(name), notify, overlay(id, + component, options), exit, reload`. `editor`, `status`, `working`, `dialog` and `grant` land with their phases. +- **Contexts.** The runtime contributes the screen name (`chat`, `sessions`, `models`; mutually exclusive), + `owner`/`guest` (from `Access`), `connected`, `overlay` and `overlay:<id>`. The active screen adds its own through + `TuiSurface.contexts()`; chat: `participant`, `editorFocused`, `editorEmpty`, `autocomplete`, `busy`. +- **Dispatch.** Commands are pre-sorted by specificity (term count), then user-bound, then registration order. The + runtime dispatches before the screen (stage 4 before 5); `ctx.overlay` wraps the component so overlay-scoped + commands run first (stage 3). While an overlay is focused only commands whose `when` names `overlay`/`overlay:<id>` + positively are eligible; everything else goes to the overlay. +- **Reports** (`/hotkeys`): launcher keys lost to other bindings, plugin defaults dropped for privileged keys, command + keys that pi-tui's transcript keys (`tui.altScreen.*` except the search-only ones) consume first, same-specificity + overlaps (`canOverlap` checks satisfiability under the exclusive groups), invalid ids/`when`s, duplicate or invalid + slash names, unknown user ids (kept, as pi does) and preference problems. +- **Ported actions.** `p2.stop` → `app.interrupt` (not while autocomplete shows, as pi's CustomEditor), + `p2.followUp`/`p2.dequeue` → `app.message.*`, `p2.models` → `app.model.select` (`/model`), `p2.tools` → + `app.tools.expand`, `p2.thinking` → `app.thinking.toggle`, `p2.exit` → `app.exit` (`!editorFocused || + editorEmpty`; `/quit`), `p2.reload` → `p2.tui.reload` (unbound, `/reload`), `p2.sessions` keeps Ctrl+R in `chat` + (`/sessions`), `app.session.resume` (`/resume`) and `app.session.new` (`/new`) unbound like pi. New: `app.clear`, + `p2.palette`, `p2.hotkeys`. The editor runs `/name args` through the slash index before sending; unknown slash text + is sent as a message. +- **Deviations.** + - The palette is **F1 only**: Ctrl+Shift+P is pi's `app.model.cycleBackward` default (non-Windows). + - Core commands come from a runtime-owned facet (`tuiCoreFacet`, loaded by the worker runtime) rather than a + plugin, like the header/footer fallbacks (7.1); they still go through `TuiCommands`, so they are rebindable, + listed and conflict-checked. + - Ctrl+N (old `p2.new`) is dropped: pi's `app.session.new` is unbound and Ctrl+N is its session-selector filter key. + - `app.model.select` and `app.session.new` require `owner` (guests cannot hold `Models` or create sessions). + +--- + +## 7. Plugin contribution model + +### 7.1 Chord wiring + +- **Registries are descriptors.** A registry is a descriptor only (`packages/chord/src/old/registry.ts:3-23`). Only objects of + lifetime "host" may call `self.register` (`packages/chord/src/old/object.ts`). Duplicate keys across objects fail under + `conflict: "error"` (`packages/chord/src/old/host.ts:606-636`). +- **Registration example:** + +```ts +// P2/plugins/artifacts/tui.ts +const Facet = local("p2.tui-artifacts", { version: 1, input: obj({}, { open: true }) }); +export default defineFacet((host) => + host.serve(Facet, (_input, _principal, self) => { + self.register(TuiToolRenderers, { artifact: artifactRenderers }); + self.register(TuiCommands, { "artifacts.list": listCommand }); + return {}; + }, { lifetime: "host" }), +); +``` + +- **Subscription.** The runtime subscribes with `host.registry(R).subscribe` (`P2/src/runtime/tui.ts`). It disposes instances whose value identity changed and recreates them. A facet reload replaces values, so + overlays and widgets from the old value are closed. +- **Fallbacks.** The built-in header and footer live in the runtime, not the registry. A plugin's `header`/`footer` + entry overrides them, and two plugins doing so is a registry conflict. + +### 7.2 Registries + +| Registry | Key | Value | Web counterpart | +| --- | --- | --- | --- | +| `TuiSlots` | `screen:<name>`, `header`, `footer`, `widget-above:<plugin>`, `widget-below:<plugin>`, `composer:<plugin>` | `(ctx) => TuiSurface` | `Slots` (`P2/src/runtime/web.tsx:47`), `SlotList` prefixes (`P2/src/runtime/web.tsx:76`) | +| `TuiCommands` | command id | section 6.3 | toolbar buttons, sheets | +| `TuiToolRenderers` | tool name | pi `ToolRenderers` shape (`CA/core/extensions/types.ts:658`) with `ctx` | **new** web `ToolCards` replacing `P2/src/web/Chat.tsx:164` | +| `TuiToolRendererResolvers` | `<order>:<plugin>` | `ToolRendererResolver` with `next()` (`CA/core/extensions/types.ts:664-667`); run in key order, then `TuiToolRenderers`, then the generic card | — | +| `TuiEntryRenderers` | entry kind | `(item, ctx) => Component` | chat-mode web rendering | +| `TranscriptEntries` (existing, shared) | entry kind | pure `(data) => EntryDisplay` (`P2/plugins/chat/contract.ts:155`) | same registry. Chat-mode's pure function moves into a module imported by both `P2/plugins/chat-mode/web.tsx` and a new `P2/plugins/chat-mode/tui.ts`. | +| `TuiMarkdownTransformers` | `<order>:<plugin>` | `MarkdownTransformer` (`CA/core/extensions/types.ts:1519`), applied in key order | — | +| `TuiFooter` | `<plugin>` | `{order, render(ctx, width)}` | sidebar-foot style | +| `TuiOverlays` | overlay id | `(ctx, args) => Component & Focusable`, with `overlayOptions` | `openLayer` sheets | +| `TuiCompletions` | **provider id** | `{triggers: string[], priority, provider}` | composer popovers (section 8) | +| `TuiThemes` | theme name | theme JSON | `P2/src/web/theme.ts` | + +Ordered registries use explicit numeric prefixes in their keys, because registry tables are maps. Several completion +providers may share a trigger character; they are consulted by priority. + +`TuiContext` holds: + +- `theme`, `keys`, `hints`; +- `scope` (`TuiScope`, `P2/src/runtime/tui.ts:43`); +- `identity` `{session, conversation}` and `navigate`; +- `editor`, `notify`, `status`, `working`, `dialog`, `overlay`; +- `grant(action, payload)`, the typed launcher bridge; +- `view` (expanded tools, show thinking). + +--- + +## 8. `@` and Tab file completion (web and TUI) + +### 8.1 How coding-agent does it + +- **Tool provisioning.** `ensureTool` (`CA/utils/tools-manager.ts:349`) finds fd/rg in `~/.pi/agent/bin` + (`CA/config.ts:639`) or on `PATH` (`CA/utils/tools-manager.ts:83-102`). Otherwise it downloads them (asset table + `CA/utils/tools-manager.ts:29-68`, download `:258`). It skips the download when offline + (`CA/utils/tools-manager.ts:361-364`) and on Termux (`:366-371`). Called at `IM:1088-1092`. +- **`@` completion** runs fd (`TUI/autocomplete.ts:149-240`) with `--type f --type d --follow --hidden` and excludes + `.git`, honoring `.gitignore`. It returns at most 100 results, which are fuzzy ranked (`TUI/autocomplete.ts:772`). +- **Tab completion** reads the local filesystem (`TUI/autocomplete.ts:600-680`). It supports `~`, absolute and parent + paths (`TUI/autocomplete.ts:549-588`). + +### 8.2 Policy for p2 + +- **Where files live.** Files live where the session runs: the server or an SSH env (`P2/plugins/envs`). +- **Scope.** Completion is **cwd-relative only**, matching the workspace confinement: `workspacePath` rejects + absolute, `~`, `..` and empty segments (`P2/plugins/workspace/protocol.ts:151-158`), and confinement is applied in + `P2/plugins/workspace/environment.ts:56-98`. +- **Out-of-scope prefixes.** Typing `~/`, `/` or `../` yields no completion. The hint says "completion covers the + session directory". `~` is never expanded with the launcher's home. +- **Later (d).** An owner-only broader mode may resolve `~` and absolute paths on the env (`hello.home`) with explicit + access checks. +- **Access.** Participants only (`canSend`, as `Workspace`, `P2/plugins/workspace/worker.ts:164-165`). + Private data directories are excluded. View-only guests get none. + +### 8.3 `FileIndex` contract (P3) + +```ts +FileIndex = remote("p2.file-index", { input: { session, conversation }, variants: { reader: { + state: { changes: num() }, // clients re-query when it advances + methods: { + complete: method({ query: string(≤200), limit?: ≤100, kind?: "any"|"file"|"dir" }, { items: {path, dir}[], truncated }), + list: method({ path: string /* validated relative dir, "" = root */ }, { entries: {name, dir}[], truncated }), + } } } }); +``` + +Served in the session worker on `WorkspaceBackend`, sharing its env, confinement, private-path exclusion and watch +(its `changes` counter). + +Index semantics: + +| Aspect | Rule | +| --- | --- | +| Files | Tracked + untracked, not ignored (git); otherwise all | +| Directories | Derived from file paths' parent prefixes. Empty directories come from `list` (a single-level `openDirReader`) or the daemon op. | +| Hidden | Included (pi passes `--hidden`); `.git` excluded | +| Ignored | Excluded where `.gitignore` applies; `node_modules` excluded in walks | +| Symlinks | Listed, not followed (no loops, no escape). pi's `--follow` is a deliberate difference. | +| Bounds | 50 000 entries, 4 MiB of listing, 10 s per rebuild, `WALK_DIRECTORIES` 5000 (`P2/plugins/workspace/environment.ts:19-30`) | +| Truncation | `truncated: true` is surfaced; the client shows "partial index" | +| Ranking | Copy of `TUI/fuzzy.ts` with a basename bonus, then shorter paths, then lexical; `kind` filter applied | +| Cache | Per conversation, rebuilt lazily when `changes` advances (debounced); worker memory only | + +Index sources, in order: + +1. **MVP (P7).** `git ls-files -z --cached --others --exclude-standard` (existing, + `P2/plugins/workspace/environment.ts:302-322`), then the bounded walk (`P2/plugins/workspace/environment.ts:323-355`). + Extend both to return directories. There is no `fd`/`rg` probing via `exec`: `command -v` is not portable to + Windows, `rg --files` returns no directories, and both are absent on most envs. +2. **Daemon op (P8, separate).** `listFiles {path, maxEntries, maxBytes, hidden, gitignore}` in the pi-env daemon, + using the Rust `ignore` crate. Work items: + - protocol doc and feature detection (`hello.version` / an `ops` list; old daemons fall back to the MVP); + - client API in `ENV/src/remote-env.ts`; + - an env-level capability on Durable's `ExecutionEnv` (an optional `listFiles` with a typed adapter, because + `ExecutionEnv` lacks it); + - a local implementation. `localDaemon()` returns only a binary path (`ENV/src/ssh.ts:362`), so local sessions need + a `Connection` to it or a Node walker with the same semantics; + - cancellation through the request context; + - conformance tests against both envs. +3. **Optional tool install.** Only if wanted for the agent's own bash: + - Normalize the daemon's Rust platform names (`ENV/docs/protocol.md:70-73`: `windows`, `x86_64`, `aarch64`) to the + tools-manager's Node names (`win32`, `x64`, `arm64`; `CA/utils/tools-manager.ts:36-64`). + - Download on the server, then upload through the daemon (`write` + `writeChunk`). + - Extract (tar/zip) on the server side, set exec permissions, and install atomically (temp name + `rename`). + - Verify the hash. + +### 8.4 Editor integration + +Shared pure module `P2/src/runtime/completion.ts`, imported by both UIs. It copies these pure parts of +`TUI/autocomplete.ts`: + +- helpers `:8-145` (`parsePathPrefix`, `buildCompletionValue`, quoting); +- `:415-547` (`applyCompletion`, `extractAtPrefix`, `extractPathPrefix`); +- forced-Tab gating `:850-859`. + +It excludes `:549-588` (homedir/statSync) and everything fs/spawn. + +**TUI.** An `AutocompleteProvider` (`TUI/autocomplete.ts:271`): + +- `@prefix` → `FileIndex.complete`; +- forced Tab on a path-like prefix → `FileIndex.list(dirname)`, filtered locally; +- `/` → commands. + +Requests are debounced (~80 ms), aborted when superseded, and inserted as `@path` exactly like pi. + +**Web** (`P2/plugins/chat/web.tsx:156-172`, plus the new-session composer): + +- **Triggers.** `@` opens the popover after a token boundary. A forced **Tab** on a plain path-like token also opens + it, using the same prefix extraction from the text before the cursor (`selectionStart`). +- **Precedence.** While the popover is open, Enter/Tab apply, arrows move and Escape closes. These win over send + (Enter, `P2/plugins/chat/web.tsx:169-172`) and stop (Escape, `P2/plugins/chat/web.tsx:164-168`). Closed: unchanged. +- **IME.** Ignore keys while `event.nativeEvent.isComposing`; never open the popover mid-composition. +- **Chat mode.** `@pi` is the agent mention (`P2/plugins/chat-mode/contract.ts:49-51`). The provider never offers + completions for a token that matches `MENTION`. While chat mode is on, file items are listed after a synthetic + "@pi — mention Pi" item. The TUI follows the same rule. +- **Both composers.** The existing-session composer and the new-session composer. Before the session exists there is + no `FileIndex`; the new-session composer offers completion only after creation, or via the env directory listing + for the chosen cwd (`Envs.listDirectory`, owner). +- **Lifetime.** The same `FileIndex` reference is held while the composer is focused. + +**Local files** never come from the worker; they come only via **L:attach** (3.2, 3.3). + +--- + +## 9. Web UI features and their TUI exposure + +| Web feature | Source | TUI exposure | +| --- | --- | --- | +| Shell, navigation, drawer | `P2/plugins/web-shell/web.tsx`, `P2/src/web/AppLayout.tsx`, `P2/src/web/navigation.ts` | Screens + overlays; Ctrl+R and `/resume` instead of a sidebar | +| Sessions grouped by env:cwd, pinned, busy, children | `P2/src/web/Drawer.tsx` | Session selector (1.9, 1.17) | +| New session (env + directory browser, model, first prompt) | `P2/src/web/new-session.tsx`, `P2/plugins/envs/DirectoryPicker.tsx` | `/new` overlay (`Envs.envs`, `Envs.listDirectory`) | +| Session options (rename, pin, archive/restore, relocate) | `P2/plugins/sessions/SessionOptions.tsx` | `/name`, `sessions.pin`, `sessions.archive`, `sessions.relocate` | +| Search, archived | `P2/plugins/search`, `P2/src/web/Drawer.tsx` | Session selector (1.17) | +| Transcript, tool cards | `P2/src/web/Chat.tsx`, `P2/src/web/Markdown.tsx`, `P2/src/web/Code.tsx` | 1.3, 1.4 | +| Composer (steer/follow-up, stop, queue edit, attachments, drag/drop) | `P2/plugins/chat/web.tsx:83-200`, `P2/src/web/composer-state.tsx` | Editor, pending area, attachments via **L:pasteImage/attach** shown as chips in a `composer:attachments` widget | +| Model/thinking sheet | `P2/src/web/ModelPicker.tsx` | 1.7 | +| Cost, context, cache, viewers | `P2/src/web/CostPanel.tsx`, `P2/src/web/SessionStats.tsx` | Footer and `/session` (1.16); viewers from `SessionMetrics.clients` | +| Owner notices, dev reload, worker retry | `P2/src/web/OwnerNotices.tsx`, `WorkerControl.retry` | Notification widget; `/retry-worker`; `/reload` (owner) | +| Logs | `P2/plugins/logs/web.tsx` | `/logs` overlay | +| Workspace (files, preview, changes/diff, commits, search, comments) | `P2/src/web/Workspace.tsx` | (d) `/files` via `FileIndex` + `Workspace.query({kind:"file"})`; `/changes` with the copied `renderDiff`; `/grep`. Comments skipped at first. | +| Workspace upload | `WorkspaceTransfer` | (d) **L:attach** with a target folder | +| Terminals | `P2/plugins/terminals` | (d) Filtered VT relay (3.7) | +| Sharing (links, reset, guests, block) | `P2/plugins/sharing/web.tsx` | `/share` (owner); copy via a C grant | +| Guest identity, read-only, presence | `P2/plugins/sharing/web.tsx`, `Access` | Header and presence after admission (4.1) | +| Chat mode, mentions, replies, notes | `P2/plugins/chat-mode/web.tsx:22`, `:65` | `composer:chat-mode` chip, `/chat-mode`, entry renderers, reply command (d) | +| Peer attribution | `P2/plugins/conversations/web.tsx` | User component prefix | +| Subagents | `P2/plugins/subagents/web.tsx` | `/agents`, header crumb | +| Artifacts | `P2/plugins/artifacts/web.tsx`, `P2/src/web/ArtifactPicker.tsx`, `P2/src/web/Viewer.tsx` | Card + `/artifacts`; text/Markdown inline; HTML/SVG via origin `openUrl` (browser sign-in, 3.5) or `download` | +| Sent files | `P2/plugins/send-file` | Card; text preview; `download` | +| Providers login | `P2/plugins/login/web.tsx` | `/login`, `/logout` (1.8) | +| Envs | `P2/plugins/envs/EnvsPanel.tsx` | `/envs` (owner): list, test, accept host key; config editing later | +| Ports | `P2/plugins/ports/PortsPanel.tsx` | `/ports` | +| Devices, sign out | `P2/plugins/devices/web.tsx` | `/devices` (owner) | +| Theme | `P2/src/web/theme.ts` | Guest-local theme | + +--- + +## 10. Phased plan + +Sizes: **S** ≤ 1 day, **M** 2–4 days, **L** 1–2 weeks. Every phase ends with `npm run check` and the TUI tests on +Node 26 (`P2/DEV.md`). Supporting server contracts always land before the UI that uses them. A command is registered +only in the phase that implements it. + +Done: P0–P11 (see git history; the "As built" sections below describe them). P8, the daemon's `listFiles`, landed in +pi 55df12bf6 and p2 52eff3371. Of P12, `Chat.shell` with `!`/`!!`, fork and clone, templates and `/skill:` (10.8) and +the Mermaid transformer (10.7) are built. Open: + +| # | Work package | Size | Acceptance criteria | Status | +| --- | --- | --- | --- | --- | +| **P12** | Later: launcher images, filtered terminal relay, optional trust-like-SSH mode, web `ToolCards` | L+ | Each behind its own design review | Open | + +### 10.1 As built: P0c and P2 + +Where the implementation differs from the text above, and why: + +- **Transcript model.** Components are keyed by the shared display items of `P2/src/web/transcript.ts` + (`buildTranscript`), not by Durable entries. The items already join calls, live slots and results and + cover p2's entry kinds (`p2.input`, chat entries, model-error notices, compaction failures). They keep their identity + while unchanged, so an unchanged item keeps its component, a streaming answer becomes its committed entry under the + same key, and only changed items rebuild (`P2/src/tui/components/transcript.ts`). +- **Reading position.** An anchor `{item key, entryId, line offset}`, not a line count from the bottom. While not + following the end, the item at the top of the viewport stays on its row when output grows, older history is + prepended or items above change height (Ctrl+O). Anchors are kept per (session, conversation) in the runtime and in + the reload snapshot; a restored anchor in older history pages towards it. +- **Overlays and reload.** The snapshot carries identity, screen, toggles, per-identity drafts, pending send IDs and + anchors (`P2/src/tui/snapshot.ts`, validated, migrating the version-1 format). The overlay stack is not carried: + overlays close on reload, which P4's criterion allows ("restores it or closes it cleanly"). +- **Sends while connecting.** Enter while the conversation reference is still connecting (after a reload or reconnect) + is held and sent once the reference is live, with the same request ID; typing goes to the editor meanwhile. A read-only + conversation refuses with a status line. +- **Ctrl+C** follows pi exactly: clear the editor, a second press within 500 ms exits. It does not abort; Escape does. +- **Status lines** sit in the dock above the editor (at most three, a repeated message collapses into a count, + cleared on navigation and on send), not in the transcript as pi's `showStatus`/`showError`. Runtime notices are not + conversation-scoped (they also occur on the sessions screen) and must not shift the transcript's reading position. +- **Header** is pi's logo with the server origin and role instead of the version, and only hints whose keys are bound. + As in pi it is the first part of the transcript and scrolls away; other screens show it above the surface. +- **Footer** (status plugin, `footer` slot): conversation stats from `pi.usage`, cache hit of the latest response, + context tokens against the owner's model catalog (guests: tokens without a denominator, and the owner-only catalog is + never requested), the session total labeled `session $x` when it exceeds the conversation's cost. The git branch + came later (10.7). +- **Tool renderers.** Shell: Durable reports the spill path as a diagnostic, not in `details`, so pi's truncation + footer is not shown; the card's own elapsed clock is shown only for runs it watched. Edit: no local preview, the + result's `details.diff`. Read: no pi-docs classification. Images: pi-tui's text fallback. +- **Code mode card** is pi's renderer (script preview, the newest eight calls with JSON arguments, output without + Durable's `Script completed|failed / Wall time / Output:` header), extended with p2's live nested slots: depth + indentation and a running call's newest output line. It replaces p2's earlier one-line summary format. +- **Plugin renderers.** `artifact`, `send_file`, `subagent` and (since P4) `ports` cards ship as `tui.ts` of their + plugins. The ports card shows forward addresses only when they match Radius's pattern, as the web card. +- **Registries** (`P2/src/runtime/tui-renderers.ts`): `TuiEntryRenderers` values are `(entry, ctx) => Component | + undefined` and replace the transcript's own display of that entry kind. The TUI also honors the shared + `TranscriptEntries` registry when a TUI facet registers it. + +### 10.2 As built: P3 (server contracts) + +- **`Chat` version 3** (`P2/plugins/chat/contract.ts`; versions 1 and 2 bind through `accepts`): + - `model` state for viewers and participants: the conversation's model from the server catalog + (`provider, modelId, name, contextWindow, reasoning, thinkingLevels`), or null. It is the guest-safe footer + denominator of section 1.16; guests still never hold `Models`. + - `compact({instructions?})` returns the Durable compaction task; `abortCompaction()` aborts this conversation's live + compaction tasks (those in `pi.live.compactions`) and returns how many were marked. Participants, so a full guest + may compact or stop a compaction, as they may send or stop. One-shot agent conversations refuse `compact`. + - `reset({handoff?})` is **owner only**: a new context changes what the agent knows for everyone in the session. + - `dequeueOwn()` withdraws the caller's queued inputs, chosen by the server-stamped author of each input's entry, in + submission order, and returns `{restored, skipped}`; inputs placed or withdrawn meanwhile count as skipped. Other + authors' inputs stay queued (they are not moved to drafts, unlike `stop`). +- **`enabledModels`** is a p2 settings.json key (`P2/src/runtime/settings.ts`) with pi's pattern format: exact + `provider/modelId` or `modelId`, `*`/`?` globs over either, optional `:level`. `Models` version 2 exposes the raw + patterns, the resolved `scopedModels` (`P2/plugins/models/scope.ts`) and `setEnabledModels` (owner). +- **`FileIndex`** is its own plugin (`P2/plugins/file-index`) rather than code inside the workspace plugin, which + another work stream owns. It consumes `WorkspaceBackend` unchanged: `query({kind:"files"})` (git ls-files or the + bounded walk, with their bounds and private-path exclusion) for the index, directories derived from file paths, + `query({kind:"list"})` for `list`. The index is cached per conversation (`FileIndexCache`, a worker local) and rebuilt + lazily when `changes` advanced, at most once per second. Ranking (`rank.ts`) is pi's `scoreEntry` (name exact 100, + name prefix 80, name substring 50, path substring 30, directories +10), then depth, length and name; p2 adds fuzzy + matches (pi-tui `fuzzyMatch`) below every substring match, since there is no fd query. `/`, `~` and `..` prefixes + complete nothing. + +### 10.3 As built: P4 (TUI on the contracts) + +- **Selectors are overlays** (`ctx.overlay`, bottom-anchored, full width), not components swapped into the editor's + place as in pi. Overlays already close on navigation, reload and plugin replacement, which gives P4's "reload during + an overlay closes it cleanly"; the model screen is gone (a version-1 snapshot on it returns to chat or sessions). +- **Models** (`P2/plugins/models/tui.ts`): pi's model selector (`P2/src/tui/components/selectors.ts`) with Tab + all/scoped, current and default badges and Ctrl+S (`app.models.save`) to set the default; pi's thinking selector + (`/thinking`, Ctrl+S `app.thinking.save`); `/model provider/id` switches at once; Shift+Tab cycles the model's + levels (from `Chat.model`); Ctrl+P / Shift+Ctrl+P cycle the scope with its `:level`, else every signed-in model. + `/scoped-models` is a reduced copy of pi's selector: toggles, Ctrl+S saves exact `provider/modelId` patterns. Without + a session a choice sets the default for new sessions. Guests (and owners before the catalog arrives) get a read-only + status line. The catalog is not refreshed from the selector (the server keeps it; login is P6). +- **Sessions** (`P2/plugins/sessions/tui.ts`): the sessions screen is pi's selector with the 1.17 adaptations. + Threaded sort nests a session's named subagent conversations (`Subagents`); selecting one opens that (session, + conversation). Message hits (`Search`, owner, debounced) form a separate "Messages (literal matches)" group. Keys: + Tab scope, `app.session.toggleSort`, `app.session.toggleNamedFilter` (pinned), `app.session.toggleArchived` + (Ctrl+A), `app.session.delete` (archive with confirmation; restore in the archived scope), `app.session.rename`, + `app.session.togglePath` (session IDs), `p2.session.pin` (Alt+P). They are registered commands whose `when` has four + terms (`sessions && selector && !overlay && !creating`), so Ctrl+D archives here instead of `app.exit`. +- **`/new [cwd]`** creates a session in the viewed session's env:cwd (or the given directory) and opens it, as pi's + `/new` starts over in place; without a session the `+ New session` prompt asks for a directory. **`/name <title>`** + renames (owner). **`/session`** (status plugin) lists the footer's numbers with their scopes (section 1.16). +- **Chat**: `/compact [instructions]`, Escape aborts a running compaction (and stops a busy run; the compaction + indicator says so), `/new-context [handoff]` (owner), Alt+Up is `dequeueOwn` (the caller's queued texts, then the + editor's text; skipped ones are reported). Chat's commands now include `!overlay`, so Escape in a selector never + stops the agent. +- **Footer**: guests get the context percentage from `Chat.model`. + +### 10.4 As built: P7a-P7c (file completion) + +- **P7a** is the P3 `FileIndex` plus tests on bounds: a directory beyond the walk's 5000 directories reports + `truncated` and still ranks within `limit` (at most 100). It does not extend the workspace plugin's git/walk to list + directories: directories derive from file paths, and empty directories appear only in `list` (Tab). +- **P7b** (`P2/src/tui/completion.ts`): `FileCompletionProvider` wraps the command registry's slash completion; `@` + asks `FileIndex.complete` (the editor debounces `@`; superseded requests are aborted through the editor's signal), + forced Tab lists the token's directory (`FileIndex.list`) and filters by name prefix, directories first (pi-tui's + `getFileSuggestions`), a single Tab match applies at once. Out-of-scope prefixes and a partial index show a status + line. The text handling (`P2/src/runtime/completion.ts`) is a copy of pi-tui's pure helpers, shared with the web. +- **P7c** (`P2/src/web/file-completion.tsx`): the session composer only; the new-session composer offers none (no + session yet; the env directory listing alternative of 8.4 is not used). The FileIndex hold starts on focus and ends + 30 s after blur. Enter and Tab apply, arrows move, Escape closes ahead of send and stop; Tab without a path-like + token keeps moving focus. IME: keys during composition are ignored and composition closes the list. `@pi` as a whole + token never completes (`MENTION_TOKEN`); the synthetic "@pi — mention Pi" item comes first while chat mode allows + replies (`canReply`, which chat-mode sets only in chat mode). Documented as a p2 addition in + `P2/docs/p2-notes.md` (Decisions). + +### 10.4c As built: P10 (settings, theme picker, rebinding) + +- **`savePrefs`** (C, `P2/src/hosts/tui-prefs-save.ts`): payload `{patch: {keybindings?, theme?, settings?}}`, `null` + removes a key, nested `settings.terminal`/`settings.markdown` merge one level. Refused before any screen + (`validatePrefsPatch`): privileged ids in `keybindings` (3.1 rule 1), any other top-level key (the editor command, + `mouse`, `downloads`), settings outside `TuiSettingsSchema` or with invalid values, invalid key ids or theme + settings, and patches that change nothing. The screen shows the file path and a leaf diff of `prefs.json` + (`- path: old` / `+ path: new`, `prefsDiff`). The run re-reads the file, applies the patch (unknown and launcher-only + keys survive untouched) and writes atomically (a fresh 0600 temp file renamed over `prefs.json`). The launcher's + watcher then posts the new preferences; a missing `~/.pi/p2-tui` is watched through its parent so a first save + applies too. pi's files are never written. +- **`/settings`** (`P2/src/tui/settings.ts`, core command `p2.settings`): pi's `SettingsList` with the **G** keys of + 1.14 (pi's labels, values and defaults) and the theme. Changes are collected and proposed as one patch when the + overlay closes. **Deviation:** no read-only server-settings section yet (no owner contract exposes + `<data>/settings.json`); the footer points to `/model` and `/session`. +- **Theme picker** (`ThemePicker`): `system`, the built-ins, custom themes from the launcher, and `automatic` (a light + and a dark theme, pi's pair setting). The preview pane renders sample Markdown, code, diff colors and a user message + with its own `ThemeRef` (`ctx.prefs.previewTheme`); the live theme changes only after the save. +- **`/keybindings`** (core command `p2.keybindings`): every command plus pi-tui's editor/transcript ids with their keys; + Enter captures the next key (`parseKey`), Backspace unbinds (`[]`), Delete restores the default (`null`), Esc cancels; + each capture is one `savePrefs`. Launcher-owned ids are listed as such and not editable. +- **Context:** `TuiContext.prefs` (`themeSetting`, `themeNames()`, `previewTheme(setting)`, `keybindings`, + `save(patch)`). +- **Tests:** `P2/test/p2/tui-settings.test.ts` (patch validation incl. privileged ids and the editor command, diff, + atomic write keeping other keys, `/settings` one patch with an isolated preview, `/keybindings` capture and the + launcher-owned refusal, end to end: diff screen, file write, watcher, the worker repainting in the new theme). + +### 10.4b As built: P9 (attachments) + +Launcher actions (`P2/src/hosts/tui-actions.ts`, section 3.6b's table gains): + +| Name | Kind | Payload / trigger | Behavior | +| --- | --- | --- | --- | +| `attach` | C | `{path?}` (`/attach [path]`; a dropped path pasted after `/attach`) | With a path: a first screen names it (the launcher resolves quotes, `file://`, backslash escapes, `~/`, relative to its own cwd). Without: the launcher's file picker (`TrustedUi.pick`: this computer's directories, filter by typing, Backspace up, Esc cancels; the worker sees nothing). Then the facts screen (path, size, type, session) and, after Enter/`y`, the upload. Result `{id, name, size, mimeType?}`, never the path. | +| `attach.key` | K `p2.attach` (unbound) | — | Picker, facts, upload; the chip arrives through `attached`. | +| `paste.key` | K `app.clipboard.pasteImage` (Ctrl+V) | — | pi's order: copied files (macOS) → facts and upload each; an image (wl-paste, xclip, native; png/jpeg/webp/gif) → a confirmation with type, size, generated name and session, then upload; else the clipboard text is pasted into the worker as a bracketed paste (controls except newline and tab removed). | + +- **Reading local files** (`openLocalFile`, section 3.3 step 7): `open` with `O_NOFOLLOW | O_NONBLOCK`, `fstat` of the + descriptor, regular files only (directories, FIFOs, sockets, devices and final symlinks are refused), the 25 MiB limit + checked first and again while streaming from that descriptor, which stays open from the facts screen to the end. +- **Upload** (section 3.4): `POST /upload?session&name` on the fixed origin with `connectionHeaders` (exact Origin and + the launcher's cookie), `redirect: "error"`, 60 s and the grant's signal (navigation and swaps cancel), a streamed + body counted against 25 MiB, an 8 KiB answer that must carry a UUID `id`. The session is the grant's target, never + a worker-supplied one. Errors stay in the launcher; the worker gets `failed`. +- **Worker** (`P2/plugins/chat/tui.ts`, `tui-commands.ts`): pending uploads per identity show as chips + (`[name size]`) in the composer line; the next send passes their ids (`Chat.send({attachments})`, text may be + empty) and clears them; Backspace on an empty editor removes the last (`p2.chat.detach`), `/detach` all; + `/attach [path]`. Pending uploads are not part of the reload snapshot (a reload drops the chips; the uploads stay on + the server unused). **Deviation:** a dropped file needs `/attach` typed before the drop (the terminal pastes the + path; the editor does not guess whether pasted text is a local path). +- **Tests:** `P2/test/p2/tui-attach.test.ts`. + +### 10.4a As built: P6 (login/logout) + +`P2/plugins/login/tui.ts` (owner; `when: owner`). `/login [provider]` lists `ProviderLogin.providers` (pi's provider +selector), then the login method when there are several, then `LoginDialog`, a copy of pi's `LoginDialogComponent` +driven by the server's `LoginState` instead of callbacks. `/logout [provider]` lists providers with stored credentials. +Both complete provider ids. One `ProviderLogin` hold spans the flow (the server cancels an unfinished login when the +hold ends); the dialog's `dispose` (any overlay close: its own, navigation, reload, plugin replacement) cancels a +running login, waits for it to settle and dismisses it. + +- **Notices.** `auth_url` and `device_code` show the address as text (and the code); `progress`/`info` as pi. + Server text is sanitized (`lineText`). Nothing is opened automatically (pi's `openBrowser` is gone): Enter (while + no prompt waits) or `app.tools.expand` (while one does) asks the launcher for `openUrl` `{kind:"external"}` (https + only, shown in full in the launcher's screen); `app.message.copy` asks for `copy` with the address (pi's key; the + launcher suspends its own Ctrl+X while the dialog has focus, rule 3). +- **Prompts.** `text` and `manual_code` use an `Input` (manual code with the web's help text, so pasting the + redirect URL works remotely); `select` a `SelectList`. `secret` uses `SecretInput` + (`P2/src/tui/components/secret-input.ts`): dots while typing, bracketed paste kept whole, no history, value wiped on + submit, never echoed (pi's `> value` line is gone), not part of the reload snapshot, and an answer error has the + value masked before it becomes a status line. +- **Cancel and dismiss.** Escape cancels a running login; once finished, Enter or Escape closes and dismisses it. +- **Infrastructure:** `ctx.overlay` now calls an optional `dispose()` of the component when its overlay closes for any + reason (`P2/src/runtime/tui.ts`, `tui-context.ts`). +- **Tests:** `P2/test/p2/tui-login.test.ts` (a remote owner on the real login server with a fixture provider: secret + never rendered, snapshotted or in status lines; OAuth with the launcher's `openUrl`/`copy` and the pasted code; + cancel, dismiss, closing ends a running login; `/logout` with nothing stored). + +### 10.5 As built: P11a-P11i (web-parity panels) + +Tests: `P2/test/p2/tui-p11.test.ts`. Panels are overlays on two generic components in +`P2/src/tui/components/list-overlay.ts`: `ListOverlay` (filter, `→` cursor, per-row actions keyed by a binding id +or a fixed panel key, footer, `follow` for log tails) and `ConfirmOverlay`/`confirm(ctx, lines)` (y/Enter, n/Escape); +`TextViewer` shows a file or diff. Each feature lives in its plugin's own `tui.ts` (the plugin that owns the +contract), so no plugin gains a dependency. + +- **P11h** (`P2/plugins/subagents/tui.ts`): `/agents` lists the main conversation, subagents and one-shot agents; + Enter navigates to (session, conversation), keeping each conversation's draft (P0c identities); Ctrl+A shows + archived ones, Ctrl+D archives/restores (participants; one-shot agents are read-only). `/main` returns. The footer + crumb is `cwd • title › <subagent>`. `TuiContext.navigate(screen, session?, conversation?)` takes the + conversation. +- **P11b** (`P2/plugins/chat-mode/tui.ts`): the reply quote and mention note come from + `P2/plugins/chat-mode/display.ts`, shared with the web facet. The `composer:chat-mode` chip (rendered by the chat + surface from `runtime.slots("composer:")`, one line under the editor) shows `Chat · <waiting>` and the hint while + on, and `[Chat off · /chat-mode]` to the owner only. `/chat-mode` toggles (owner). `/reply` picks the message the + next send answers (the web's hover button / long press); the pending area shows "↪ Replying to …", and `/reply` + again cancels. Replies and the first `@pi` item exist only while chat mode is on (the web's `canReply`). +- **P11d** (`P2/plugins/logs/tui.ts`, `P2/plugins/status/tui.ts`): owner notices are status lines (pending ones + summarized once, then one line per new notice); `/notices` lists them and Ctrl+D dismisses. `/logs [source]` opens + the viewed session's worker log (else the server's), follows the tail and Tab cycles sources. A failed session + worker (`SessionStatus.error`) is reported once with `/retry-worker` (`WorkerControl.retry`). **Deviation:** dev + reload is `/dev-reload`, not `/reload` (that is the TUI's own reload, P0c). The dev channel `/__p2_dev` is an owner + HTTP route, so the command asks the launcher for a C action `devCommand` (`{action: "reload"}`). The launcher action + exists since P5 (3.6b). +- **P11e** (`P2/plugins/envs/tui.ts`): `/envs` is the one environment screen. Enter moves the viewed session to the + selected environment (section 10.6). Management is on secondary keys: + - Ctrl+T tests and the list shows the last test. + - Ctrl+K reviews an unverified or changed host key in a confirmation with the fingerprints and sends exactly the + test's `lines`. + - Ctrl+F forgets a key (also confirmed). + - Config editing, key generation and upload stay on the web. +- **P11f** (`P2/plugins/ports/tui.ts`): `/ports` lists forwards with Radius addresses only (the card's `FORWARD` + pattern; other server-reported URLs are not shown). Ctrl+P makes one public after a confirmation (private at once), + Ctrl+D closes, `/ports open <port> [name] [public]` adds (public confirmed first). Opening an address can use P5's `openUrl` + (`{kind: "external", url}`). +- **P11g** (`P2/plugins/devices/tui.ts`): mint a one-time token (only when `canMint`, with the web's warning). Signing + out is a C launcher action `signOut` (`{all}`): the cookie lives in the launcher, and its consent UI is the + confirmation. The launcher action exists since P5 (3.6b): it calls the sign-out route with its cookie; the refused reconnect then + forgets the cookie and asks for an owner link. +- **P11i** (`P2/plugins/workspace/tui.ts`, a new file in impl-p2b's plugin; nothing of theirs changed): `/files + [dir]` browses (`Workspace.query list`) and shows files (`file`), `/changes` lists `status` and shows a change's + diff, `/grep <text>` searches and opens the match at its line. All go through `Workspace` (the guide said + `FileIndex` for `/files`; the workspace listing carries sizes and ignored flags and is one reference for the three). + Participants only (`when: participant`), so view-only guests never acquire `Workspace`. Git's unified patch is + converted to pi's numbered diff lines (`piDiff`) for the copied `renderDiff`. Worker text is sanitized line by line + (`lineText`). Comments, commits and uploads stay on the web. +- **Owner panels** (`/logs`, `/notices`, `/retry-worker`, `/dev-reload`, `/envs`, `/ports`, `/devices`, `/share`) + are `when: owner`; the notice and worker-status watches start only for owners. A test asserts that guests acquire + none of them. +- **P11a** (`P2/plugins/sharing/tui.ts`): `/share` (owner) is the web's share sheet: the read-only and read/write + links and the guests connected now (live from `Sharing.guests`, which is the presence UI on the owner's side). + Enter copies a link through the launcher's `copy` (`{text, label}`; created first when missing), and a first + read/write link on a remote environment is confirmed in the worker before it exists (the web's "Guests can run + commands on …"). Ctrl+R resets a link and Ctrl+D removes a guest; both are confirmed. The TUI never shows a link + itself: only the launcher's consent screen does, on the owner's terminal. On the guest's side, the header (P0a) + shows name, "(unverified)" and access. A view-only guest gets "Joined as <name> (unverified) · read-only" where the + editor would be (the web's read-only bar). GuestPresence is still held from the `widget-above:sharing` slot. +- **P11c**: + - **Artifacts** (`P2/plugins/artifacts/tui.ts`). The card adds "/artifacts opens". For text and Markdown it shows + the content the call wrote: 8 lines, all when expanded. + - **`/artifacts`** lists `ArtifactList`. Enter shows an artifact inline: Markdown rendered with the instance + theme, text and HTML/SVG source as sanitized lines. HTML/SVG carry a "Source only" note, because they run only in + a browser. Ctrl+B is `openUrl` `{kind: "origin", path: "/artifact?…"}` and Ctrl+S is `download` + `{kind: "artifact", …}`; the panel and the launcher's consent screen both say the browser needs its own sign-in. + - **Sent files** (`P2/plugins/send-file/tui.ts`). The card adds "/downloads saves". + - **`/downloads`** lists the `send_file` results of the loaded transcript (`ChatSurface.toolViews`, the web card's + checks: a UUID snapshot ID, a name, a size). Enter previews text, Ctrl+S is `download` `{kind: "file", …}` and + Ctrl+B is `openUrl` on `/download?…`. + - **Deviation: two new worker contracts.** The TUI worker has no cookie, so it cannot read `/artifact` or + `/download` for inline views. Both contracts are new objects, so no existing contract changed: + - `ArtifactContent.read({name, version})` (`P2/plugins/artifacts/contract.ts`): at most 512 KiB of text, + `truncated` otherwise. + - `SentFiles.preview({id})` (`P2/plugins/send-file/contract.ts`): the first 256 KiB as text, or `null` for + binary (NUL or invalid UTF-8). + - Both are served by the session worker from the data directory, with the routes' authority (`canView` on the + session). Tests are in `P2/test/p2/artifacts.test.ts` and `P2/test/p2/send-file.test.ts`. + - Bounding and never overwriting downloads are the launcher's job (3.6b). +- **Launcher results** (`P2/src/tui/grant-notice.ts`): `ok` shows the action's notice and `denied` (declined) shows + nothing. `failed` says the launcher showed its own message, since the worker never learns it. `busy` and + `cancelled` are named. `/dev-reload` shows the server's `{message}` (e.g. "No dirty plugins"). `/devices` sign-out + of "this device" on loopback is refused by the launcher (`denied`, "Not signed out"), because the device owner has + no cookie. +- `ListOverlay` footers wrap (at most three lines each), and `TextViewer` passes the width to its lines, so Markdown + wraps. + +### 10.6 As built: working directory and environments (`/cd`, `/new [env:]path`) + +Tests: `P2/test/p2/tui-location.test.ts`. Code: `P2/plugins/sessions/tui-location.ts`, `P2/plugins/sessions/tui.ts`. + +- **Form.** A location is written as the footer writes it, `[env:]path` (`gpu:/data`). No prefix is this device for + `/new`, and the session's own environment for `/cd` (cd semantics). `local:` names this device, unless an + environment is called "local". A prefix naming no configured environment is refused ("Unknown environment …") + unless it is one letter, which reads as a Windows drive. Relative paths join the session's directory for `/cd` + (`..` and `.` resolved, either separator style), and the home directory otherwise (`~/rel`). The server expands `~` + on the target machine. +- **`/cd [[env:]path]`** (owner, the web's Relocate in the session sheet): + - It resolves and checks the location with `Envs.listDirectory`, as the web's DirectoryPicker does before "Use", then + calls `Sessions.relocate` with the listed (canonical) path. + - Moving to another environment is confirmed with the web's note ("Tools will run on X. The history stays; files + are not copied."). + - Errors are the server's message, as on the web (ENOENT, SSH failures, "Stop the current run before changing + directory"). + - The footer follows the catalog. + - Without an argument, a prompt opens with the environment choice. Its note stands above the input, and Enter is + the confirmation, like the web's "Move session to" button. +- **Completion.** Tab completes the argument of `/cd` and `/new`: environment names (`gpu:`), then the typed + directory's subfolders from `Envs.listDirectory` (dot folders only after a typed dot). **Deviation:** not + `FileIndex`, which is confined to the session directory and its environment and cannot reach parents, `~` or another + environment. +- **Environments.** The "+ New session" prompt (sessions screen) and the `/cd` prompt choose the environment with Tab: + this device, then each configured environment, starting at that machine's `~` (the web picker's environment select + browses `~`). The first use acquires `Envs` and calls `warm` (the web's SSH warm-up). The reference is held while in + use and released 30 s after the last request. Guests never acquire it (tested). +- **`/new [[env:]path]`** creates there (`/new gpu:/data`); without an argument it creates in the viewed session's + env:cwd as before. A bare path used to take the default environment; it now means this device, matching the footer + form. +- **`/envs` moves sessions** (the owner's "we only need envs"): + - **Screen.** The first row is "This device", then the configured environments. The viewed session's environment + is marked "●" and "this session", and selected when the screen opens. + - **Enter** moves the session there: the directory prompt opens on that environment. It is prefilled with the + session's directory when it exists there (checked with `Envs.listDirectory`), else with that machine's home (its + listed path). An unreachable environment's error shows under the input. + - **Confirmation.** The prompt shows the move note ("Tools will run on X. The history stays; files are not copied."), + and Enter confirms, as in the `/cd` prompt. Then `Sessions.relocate`. Escape moves nothing. + - **The session's own environment:** Enter opens the same prompt without the note, so it works as `/cd`. + - **Wiring.** Moving belongs to the sessions plugin, which already imports the envs contract, so `/envs` reaches it + through a host registry: `TuiSessionPlaces` (`P2/src/runtime/tui-sessions.ts`, `where` and `moveTo`). Without the + sessions plugin, `/envs` only manages environments. + - **Not changed.** `/cd [[env:]path]` stays as a shortcut. + - The web DirectoryPicker port (folder browser) asked for earlier is not built yet. +- **/hotkeys** shows each command's argument hint (`/cd [[env:]path]`) and, for owners, a "Working directory and + environment" section: Tab in the prompts, the `[env:]path` form, `/cd` rules, and what Enter and the secondary keys + do in `/envs`. +- **Prefilled inputs.** pi-tui's `Input.setValue` keeps the cursor where it was (0), so typing after a prefilled + directory or title went in front of it. The prompts now fill the input by typing it (`fillInput`). +- **Command completion** (`P2/src/runtime/tui-commands.ts`): + - `slash.tabOnly` makes argument completion run on Tab only, never while typing, so Enter submits a typed path + instead of accepting a listed subfolder. + - A forced Tab inside a command's argument now asks the command's `complete` (pi's argument completions). + `P2/src/tui/completion.ts` falls back to file completion when the command has none. + +### 10.7 As built: footer branch, Mermaid (P2 leftover, P12) + +- **Git branch** (`P2/plugins/workspace/tui.ts` `GitBranch`, slot `footer:branch`): + - The footer shows pi's `cwd (branch)` and "detached" for a detached HEAD, as pi does. + - The branch comes from `Workspace.query({kind: "status"})`, asked when the conversation opens and again when + `Workspace.changes` advances (500 ms debounce, one query at a time). + - It is asked only by those who may hold `Workspace`: the owner and full-access guests. View-only guests are refused, + so they never ask. + - The status plugin reads it through `runtime.slots("footer:")` and does not depend on the workspace plugin. + - Tests: `P2/test/p2/tui-p11.test.ts` (main, then a branch switch after a file change, then detached). +- **Mermaid** (`P2/src/tui/components/mermaid.ts`, registered by the core facet as `00:mermaid`, ahead of plugin + transformers, as pi puts it before extension transformers): + - A copy of pi's transformer. The mode comes from the visitor's `markdown.mermaid` setting (default `streaming`, as + pi) and the colors from the instance theme. Thinking blocks are never transformed, too-wide diagrams keep their + source, and invalid ones say why once complete. + - **Deviation:** `MarkdownTransformContext` carries `theme` and `settings`. pi's transformers close over its global + theme and settings manager; p2's are registry entries shared by all instances (5.4). + - `grok-mermaid` 0.2.3 was checked to be pure (no dependencies; no fs, process, network or eval) and added to p2's + dependencies. The lockfile changed only in p2's dependency list (the package was already installed for + coding-agent). + - Tests: `P2/test/p2/tui-mermaid.test.ts`. The sandbox test in `P2/test/p2/tui-render.test.ts` renders a diagram + inside the real permission-dropped worker. +- **`/envs` warm-up.** Opening the screen calls `Envs.warm`, as the web's directory picker does. + +### 10.8 As built (P12): shell, skills and templates, fork and clone + +#### `Chat.shell`: `!cmd` and `!!cmd` (as built) + +Tests: `P2/test/p2/shell.test.ts` (a bare session host with real Durable and a real local environment), +`P2/test/p2/tui-shell.test.ts`. + +- **What pi does** (`IM:3346-3360`, `CA/core/messages.ts:30`): + - `!cmd` runs a shell command and shows a bash execution component with its output streaming. + - The result becomes a `bashExecution` message. It is in the model's context ("Ran `cmd`" plus the output) but starts + no run. `!!cmd` is the same, excluded from the context. + - One command at a time; Escape cancels. +- **Contract.** + - `Chat` is version 4 (versions 2 and 3 are accepted as they are). + - Participants get `shell({command, includeInContext, requestId})` → `{shellId}` and `abortShell({shellId})` → + `{aborted}`. + - Both variants get the state `liveShell` (a state member cannot share the method's name): + `{id, command, author, includeInContext, output, truncated, running, exitCode?, cancelled, error?} | null`. + - **Who may:** anyone who may send, so full guests too, as tools run commands for them. Viewers watch. + One-shot agents' conversations are refused (the `ReadOnlyConversations` check). + - **Cancelling:** the command's author or the owner. + - **One at a time per conversation;** a second command is refused with pi's text (`SHELL_BUSY`). + - **Retries:** a call with the same request ID runs once. +- **Worker** (`P2/plugins/chat/worker.ts`; the shared `Conversation` object runs it, `runShell`/`abortShell`): + - The command runs on the conversation's environment, `Harness.environments.at(agent.cwd)`: local or remote, and + a subagent's own cwd. + - It uses the bash tool's settings (`inheritEnv`, spill and an output window at Durable's 2000 lines / 50 KiB, + sampled every 200 ms at 100 KiB/s). + - The kept tail is published as `liveShell`. + - It runs as one active call (`Liveness.call`), so the worker stays up, and a move or clone waits for it. It runs + under its own root context (the method returns at once). Closing the object cancels it. + - When it ends, a Durable write submission places a `p2.shell` entry (`P2/src/runtime/shell.ts`, `ShellData`: + author, command, output tail, `exitCode`, `cancelled`, `truncated`, `fullOutputPath`, `includeInContext`, `error`). + - **`!`:** the entry also carries a user message (the speaker block plus pi's `bashExecutionToText`, + `shellContextText`). The agent reads it on its next run, and no run starts now. + - **`!!`:** the entry is display-only. + - **Busy agent:** while the agent runs, the entry is placed at the run's next boundary, like any write. + `liveShell` clears once it is placed. + - The data has no `text` field, so no reader takes it for a person's message. + - **Deviation:** a command is not refused during a compaction; the write's placement keeps the order. +- **TUI** (`P2/plugins/chat/tui.ts`): + - **Bash mode:** `!` text turns the editor border to `bashMode`. Enter on `!cmd`/`!!cmd` calls `Chat.shell`; a lone + `!` is an ordinary message. + - **Live command:** a copy of pi's `BashExecutionComponent` (`P2/src/tui/components/bash-execution.ts`: instance + theme, sanitized output, loader only while live, the author named for guests' commands). It is the transcript's + new `tail`, at the end like pi's chat flow. + - **Escape, in pi's order:** the running agent, then the running command, then bash mode's text (cleared), then the + double Escape. + - **Entries:** finished `p2.shell` entries render with the same component, through an entry renderer registered by + the core facet. The entry kind is the host's. + - **Why the core facet:** registering it from the chat facet made a reload of that facet rebuild the renderers + mid-reload, and Ctrl+End stopped following (`tui-render` "reading position"). That rebuild problem remains for + any facet that contributes entry renderers. +- **Web:** the entry is a plain notice: a marker `$ command` whose detail is the output in a fenced block, how it + ended, and who ran it (`P2/src/web/transcript.ts`). There is no live view and nothing to start commands. + +#### `/skill:<name>` and prompt templates (as built) + +- **pi's behavior, copied** (`P2/plugins/prompt/templates.ts`, from `CA/core/prompt-templates.ts`, + `CA/utils/frontmatter.ts` and `CA/core/agent-session.ts` `_expandSkillCommand`; no runtime imports): templates are + `.md` files with optional frontmatter `description` (else the first line, 60 characters) and `argument-hint`, used + as `/<name> args`; arguments split bash-style with quotes and substitute `$1`, `$@`, `$ARGUMENTS`, `${N:-default}`, + `${@:-default}`, `${@:N}`, `${@:N:L}` without recursion. `/skill:<name> args` becomes + `<skill name location>References are relative to <dir>.\n\n<body></skill>` plus the args. Skill expansion comes + first, then the first template of that name; unknown names and unreadable skills stay as typed. Skills hidden from + the model (`disable-model-invocation`) are still invocable, as in pi. +- **Locations** (the session's environment, like skills): `<data>/prompts` locally or `~/.pi/p2/prompts` on a remote + environment (global, listed first, so it wins a name clash), then `<cwd>/.pi/prompts` (project). Only files directly + in those directories; symbolic links are not followed; an unreadable or unparsable template is skipped. They are + loaded with AGENTS.md and skills (`loadPromptResources`, `P2/plugins/prompt/context-files.ts`) and the prompt + observer watches both directories, so changes reload live. +- **Contract** (`P2/plugins/prompt/contract.ts`): `PromptResources({session})`, served by the session worker to + participants only (`canSend`; viewers are refused). State `templates: [{name, description, argumentHint?, source}]` + and `skills: [{name, description}]`; method `expand({text})` → `{text, skill?: {name}}`, which reads skill files + through the session's environment when invoked. Decision on the open point: a separate method, not a `Chat.send` + flag, so both clients use it and `send` stays unchanged. +- **TUI** (`P2/plugins/prompt/tui.ts`): a slot surface (no rendering) holds `PromptResources` of the viewed session + for owners and full-access guests, and contributes a `TuiSlashSources` entry (`P2/src/runtime/tui-commands.ts`). + Slash completion lists templates (with their argument hint) and `skill:<name>` after the commands, matching skills + by their bare name first (pi's order); an entry with a command's slash name is neither listed nor expanded. On + Enter, the chat editor expands a `/template` or `/skill:` through `ctx.commands.expand` (the worker method) and sends + the result (one small hook in `P2/plugins/chat/tui.ts`). A user message that is a skill block renders as pi's + collapsed `[skill] <name>` block (`P2/src/tui/components/skill-invocation.ts`, a copy of + `CMP/skill-invocation-message.ts`), expanded with the tool toggle (Ctrl+O), followed by the user's own text. +- **Web**: the composer asks the `ComposerExpanders` registry (`P2/src/runtime/composer.ts`) before a send; the + prompt plugin's web facet (`P2/plugins/prompt/web.tsx` → `composer.ts`) expands through the same worker method. + No completion UI in the web; the new-session composer does not expand (its session does not exist yet). +- **Tests:** `P2/test/p2/prompt-templates.test.ts` (substitution and parsing, the loader, the worker object with live + reload, expand, the viewer refusal and the web expander, the TUI completion and expansion through a worker stub, + the skill block component, and a real worker driven through the launcher). + +#### `/fork` and `/clone`: `Sessions.fork` and `Sessions.clone` (as built) + +Tests: `P2/test/p2/fork-clone.test.ts` (real server and workers, fake provider), `P2/test/p2/tui-fork.test.ts`. + +- **What pi does:** + - `/clone` duplicates the current session at its current position. + - `/fork` opens a selector of the user's messages. The new session holds the history before the chosen message, and + the editor receives that message's text. + - `doubleEscapeAction: "fork"` opens the same selector. +- **Contract.** `Sessions` is version 3 (version 2 is accepted as it is). The owner gets `clone({id})` → `Session` + and `fork({id, conversation, before})` → `Session`, where `before` is the chosen message's entry. +- **Open questions, decided conservatively:** + - The server copies the store. + - Forking a subagent conversation is refused ("Forking a subagent conversation is not supported yet"). +- **`clone`** (`P2/plugins/sessions/server.ts`): + - It runs inside `SessionHosts.pause` (the relocation machinery, now generic; `relocate` uses it). A running worker + is fenced first (`WorkerBoundary.prepare`), and refused while it works ("Stop the current run before cloning"; + a running `!` command counts). + - With the worker stopped, the store is copied as files (`sessions/<id>.sqlite`, `-wal`, `-shm`: closed, they + belong together), together with `artifacts/<id>`, `downloads/<id>` and `uploads/<id>` (`P2/plugins/sessions/copy.ts`: + 0700 directories, 0600 files, links skipped, nothing overwritten). The worker then restarts if it ran. + - **Deviation:** a file copy of the stopped store instead of SQLite's online backup. + - **The catalog entry comes last,** under the copy's pre-chosen ID (`Catalog.create` takes an optional `id`): + same cwd, env, model and level, title "<title> (copy)". A failed copy is removed. + - Subagent conversations come along in the store. Their sidebar entries appear when the copy's worker reports + them (`ChildStore`); nothing to copy there. +- **`fork`:** + - **Export.** The source worker exports the history through `SessionTransfer.history` (a new server-only object, + `P2/plugins/sessions/contract.ts`). It is the Durable context as of the entry before `before` (`context({at})`), + as entry drafts: kind, data, the model messages with context edits applied (`contributions`, so no edit refers to + an old ID), and `head: "self"` on a leading head marker. A fork thus holds the active context, not entries before + a compaction. + - **Seed.** The server copies the per-session directories, creates the entry ("<title> (fork)"), and the new + session's worker writes the drafts into conversation 1 in one commit (`SessionTransfer.seed`, refused when it has + history) before the ID is returned. + - Forking before the first message gives an empty session. A failed seed archives the new entry. + - **Deviation:** a fork's `/artifacts` index (a Durable document) is not rebuilt; its artifact cards work, because + the files are copied. +- **TUI** (`P2/plugins/chat/tui.ts`, `tui-commands.ts`; in the chat plugin because the picker needs the loaded + transcript and sessions cannot import chat): + - **`/fork`** (`app.session.fork`, owner) opens a copy of pi's `UserMessageSelectorComponent` + (`P2/src/tui/components/user-message-selector.ts`) over people's messages in the loaded transcript, newest + selected. Enter forks, opens the new session with the message in its editor (its draft), and says "Forked to new + session". + - **`/clone`** (`p2.session.clone`, owner) opens the copy ("Cloned to new session"). + - **Double Escape:** `doubleEscapeAction: "fork"` opens the picker on a second Escape within 500 ms on an empty, + idle editor (owner). `"tree"` behaves as `"none"` (no `/tree`). +- **Web:** no UI yet. + +--- + +## 11. Copy checklist (dependency removal) + +Every copied file must pass this checklist and a smoke test **inside the real permission-dropped worker**: bundle it, +render it in `P2/test/p2/tui-launcher.test.ts`'s sandboxed session, and assert that no `Unavailable in the TUI sandbox` / +permission errors occur. + +General removals: + +- `node:fs`, `node:child_process`, `node:os` homedir/platform, `process.env`, `process.platform`; +- clipboard, browser opening and the external editor; +- `ModelRuntime`, `SessionManager`, `AgentSession` and `SettingsManager`; +- global `theme` and `keyHint`; +- `CA/config.ts` path helpers. + +Inject data and actions through constructor parameters. Copy pure helpers. + +| File | Remove / replace | +| --- | --- | +| `CMP/session-selector.ts` | `spawnSync` `trash`, `existsSync`, `unlink` (`CMP/session-selector.ts:1-4`, delete at `:654-687`) → `Sessions.archive`; `canonicalizePath` (`:18`, `:51-53`) and `os` homedir (`:4`, `:27`) → server-provided display paths; `SessionInfo` → p2 rows | +| `CMP/session-selector-search.ts` | Keep the pure parsing; drop regex/phrase support or keep it for local title filtering only (1.17) | +| `CMP/extension-editor.ts` | `editInExternalEditor` and `process.env.VISUAL/EDITOR` (`CMP/extension-editor.ts:17`, `:59-61`) → launcher `externalEditor` grant | +| `CMP/login-dialog.ts` | `openBrowser` (`CMP/login-dialog.ts:3`, `:99-110`) → `openUrl` C grant; `process.platform` hint (`:124`); the `> value` echo (`:80-83`) → secret mode | +| `CMP/auth-url.ts` | `copyToClipboard` (`CMP/auth-url.ts:2`, `:31-34`) → `copy` C grant; platform click hint (`:26`) → guest platform from `capabilities` | +| `CMP/model-selector.ts` | `ModelRuntime` and `refreshModelCatalogs` (`CMP/model-selector.ts:12-13`, `:59`, `:76-94`, `:153-158`, `:162-208`) → injected model list, current/default and a refresh action (`ProviderLogin.refresh`) | +| `CMP/tool-execution.ts` | `ensurePngTranscoder`; `MouseRegion` kept only with mouse mode; global theme (5.3) | +| `REN/read.ts` | `node:path` cwd resolution, `getReadmePath`, `resolveToCwd` → display relative to the session cwd from `Sessions` | +| `REN/edit.ts` | `computeEditsDiff` (local fs) → `details.diff` only | +| `CA/core/tools/render-utils.ts` | `node:os` homedir, `pathToFileURL` hyperlinks, `resolvePath` → plain relative display | +| `CMP/footer.ts` | `AgentSession`, `FooterDataProvider`, `process.env.HOME` → 1.16 data | +| `CMP/bash-execution.ts` | local output paths; keep the rendering | +| `THM/theme.ts` | Section 5.3 | +| `CMP/custom-editor.ts` | `AppKeybinding` typing → command dispatcher; paste-image hook removed | diff --git a/apps/p2/docs/p3-functions/README.md b/apps/p2/docs/p3-functions/README.md new file mode 100644 index 000000000..db5de601e --- /dev/null +++ b/apps/p2/docs/p3-functions/README.md @@ -0,0 +1,74 @@ +# p3 functionality map + +What the app must do, independent of p2's plugin layout. Step 1 of deriving p3's objects (`../redesign-decisions.md`). +The detailed tables per area are in this directory: `conversations.md`, `sessions-access.md`, +`environments-workspace.md`, `models-prompt-shells.md`, `core.md`. This page condenses them. + +## Domains + +| # | Domain | Main functions | +|---|---|---| +| 1 | Sessions | create (local or remote, directory check, defaults); live list (pinned, grouped by machine and directory) that never starts a worker; rename, automatic title, agent rename; pin; archive and restore (stop worker, end guest access, close forwards); relocate (idle only, fenced, followers move, cross-machine refusal); clone; fork at a user message; delete (new); unread marks | +| 2 | Session workers | start on demand; one worker per store; idle stop after 5 min without viewers, calls or tasks; sticky start failure with retry; generation fence on reports; crash reports; shutdown | +| 3 | Conversations and agent | live view window; history paging and retained pages; jump to an entry; context, cache and cost; working indicator; tool cards and nested calls; send, steer, queue, dedupe per author request id; input gate until prompt resources load; structured attribution; stop and dequeue with restored drafts; model and thinking (owner); compact and reset; read-only conversations; model error entries; resume interrupted work | +| 4 | Shell commands | `!cmd` / `!!cmd` in the conversation's cwd and environment; one at a time; live tail; spill file; result entry placed at the next boundary; cancel; dedupe | +| 5 | Chat mode | per-conversation switch; chat messages without a turn; `@pi` mention with a digest; reply quotes | +| 6 | Agents | named subagents (spawn, steer, configure, archive, report answers once, own cwd); one-shot agents for scripts (read-only, JSON result); messages between sessions (agent tool, attributed, idempotent) | +| 7 | Code mode | scripts calling tools as nested durable calls; replay after restart; switch on and off | +| 8 | Session files | attachments (≤25 MiB, copied to remote environments); sent-file snapshots; versioned artifacts with sandboxed preview; shell spill files; previews; ranged downloads; copy on clone; delete with the session | +| 9 | Environments and execution | environment config (validation, keys, host keys, test); directory browsing local and remote; connection pools; daemon deployment; execution environment per session and per conversation cwd; reopen after a move; OS probe (PowerShell) | +| 10 | Workspace | list, read, search, git status, diff, log, commits; file index and `@` completion; change notifications; uploads into the project; downloads; review comments sent as a message; confinement and git hardening | +| 11 | Terminals | detached host process owning PTYs; attach with a screen snapshot; several viewers; flow control; single-use channel tokens; host version notice | +| 12 | Access | owner on loopback, device cookies, sign-in tokens, sign out and rotate; share links per mode, join, unique guest names, grant revision, remove, reset; origin checks; proxy trust; revocation of live access | +| 13 | Models, credentials, settings | model catalog (custom providers, cache); credentials (locked, never to clients); API key and OAuth login with prompts and notices; default model and scope; run settings; code mode switch; owner name; hot reload with last valid state | +| 14 | Prompt resources | AGENTS.md, skills, templates read in the session's environment; watched; expansion before send | +| 15 | Status and presence | per-session status (busy, tasks, usage, error); viewer counts; connected guests; keep-alive inputs | +| 16 | Search | message index per session with catch-up cursors; title and text search; open a hit | +| 17 | Remote access and forwards | Radius relay link; relayed owner treated as remote; forwards (≤50, public toggle, agent tool, closed on archive) | +| 18 | Logs and notices | capture and redact; bounded rings; detached process snapshots; owner notices; dismiss | +| 19 | Dev and reload | dirty tracking; plugin reload; supervised restart; bundle publication; client update detection; `reload_plugins` tool | +| 20 | Web shell | layout, navigation stack, composer state, theme, reconnect, installable app | +| 21 | TUI shell | launcher sandbox; consent actions (copy, upload, download, open URL, editor); preferences; commands; snapshot swap; terminal safety | +| 22 | Transport | frames and limits; reconnect and resubscribe; protocol refresh | + +## Cross-cutting needs + +| Need | Where | +|---|---| +| One session file store (caps, immutable ids, ranges, previews; copied and deleted with the session) | attachments, sent files, artifacts, spill files, workspace downloads | +| Per-session data lifecycle across hosts (archive, clone, delete) | store, files, links, guests, index rows, read marks, children, forwards | +| Reading without starting a worker | session list, status, children, unread, artifact list, logs, cost sheet | +| Worker-to-server reports with a generation fence | status, children, index feed | +| Catch-up after commits (cursor plus wake) | search indexer, subagent reporter | +| Work that outlives its caller, with cancel, dedupe and an outcome | shell, relocate, clone, fork, login, peer send, scripts | +| Idempotency per author request id | send, write, shell, peer send | +| Cancellation chains | script → nested calls → child conversation; stop; login abandon | +| Keep-alive versus presence | viewers, lingering tabs, active calls, tasks | +| Execution environment that follows the session; file watching in it with poll fallback and last valid state; connection pools | workspace, file index, prompt, PowerShell, terminals, browsing | +| Single writer for configuration files; central redaction; credentials never to clients | settings, models, auth, radius, envs | +| Attribution of input (structured author, peer, owner-initiated checks) | send, shell, peer messages, privileged tools | +| One permission matrix (owner, guest read, guest write, worker; terminals owner only; workspace not for guest read) | every function | +| Revocation of live access | guests, devices | +| Chunked byte transfer with ranges | uploads, downloads, attachment copies | +| One notice channel | logs, dev, model errors, grants, failures | +| UI state that survives navigation, reload and reconnect | drafts, pending sends, reading positions, review drafts | +| TUI consent for local side effects | copy, upload, download, open URL, editor, sign out | + +## Duplicated today (one function, several implementations) + +Input gate loop (4); latest answer text (3); download snapshot writers (2); upload handling (2 routes, 2 caps, 2 layouts); +size and id validation (3); environment config parsing and SSH options (3); SSH connections to an environment (4); +confinement and private-directory denial (3); full file listing (2); path resolution (4); "archived?" checks (5); +presence leases (3); generation-fenced report stores (2); commit catch-up loops (2); revocation fan-out (3); thinking +clamp and model lookup (4); watch plus last valid state (5); secret registration for redaction (5); notice channels (5); +staleness checks (3); reconnect backoff (3); drafts (3); following the session location (5); "offer the tool only +when" switches (3); themes and transcript renderers (web and TUI). + +## Candidates to drop + +Legacy speaker and attribution parsing; peer attribution as a text prefix; shell abort after 30 s without a viewer; +catalog and cookie migrations; guest last-seen; the subagents copy of conversation config; warming every environment +connection when the folder picker opens; duplicate environment test state; the worker's unused environment watcher; old +relay protocol versions; the proxy-marker object; unused snapshot methods; the fake provider in product code; polling +log snapshot files; "removed mid-script"; `P2_DEV_RESUME`; old TUI snapshot formats; legacy default-model environment +aliases. diff --git a/apps/p2/docs/p3-functions/conversations.md b/apps/p2/docs/p3-functions/conversations.md new file mode 100644 index 000000000..648493382 --- /dev/null +++ b/apps/p2/docs/p3-functions/conversations.md @@ -0,0 +1,175 @@ +# Functionality map: conversations, agent loop, chat mode, drafts, cross-session messages, sent files, artifacts + +Read-only, based on origin/private `5abe3f6d6`. Abbreviations: +- **who:** O owner, GR guest who can read, GW guest who can write (full grant), W session worker, S system. +- **where:** srv server, wkr session worker, web, tui, env (remote execution environment). +- **durability:** eph ephemeral, rst until restart, pers persisted, tx transcript-bound. + +## Viewing a conversation + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| conv | View a conversation live: newest entries (≤100 or 512 KiB, at least 20) plus all conversation docs | entries ~KB each; window up to 512 KiB | tx | O, GR | wkr→web, tui | watch conversation | +| conv | Window start stays put while within 1.5× the limits, so new entries arrive as small appends | previous window start | eph | S | wkr | view | +| conv | Page older history: 100 entries / 1 MB / 2000 scanned per page; compaction and reset markers always included | pages ≤1 MB | tx | O, GR | wkr | view | +| conv | Keep entries that a compaction or reset dropped from the view; client keeps pages of 4 conversations | client cache | eph | O, GR | web, tui | history paging | +| conv | Jump to an entry (search hit, quoted message), loading older pages until it is shown, then highlight it | entry id | eph | O, GR | web | history paging, search | +| conv | Context usage (estimated tokens vs. model window), latest response's cache usage | numbers | tx (derived) | O, GR | wkr→web, tui | model metadata | +| conv | Cost sheet: each response's cost, cache hit, provider, model, stop reason | ~100 B per response | tx | O, GR | wkr→web, tui | view | +| conv | Model metadata for viewers: name, window, thinking levels; no credentials | small | eph | O, GR | wkr | model catalog | +| conv | Working indicator: retry countdown, deferred response, compaction (with reason and attempt), running tool, working | live doc | tx | O, GR | web, tui | view | +| conv | Render the transcript: thinking (toggle), per-tool cards (bash, codemode with its nested calls, read, write, edit diff, subagent, artifact, ports, send_file), inline images, failures, markers, subagent reports | entries | tx | O, GR | web, tui | renderer extensions | +| conv | Show a tool cut short by a restart as "interrupted" (not failed) | error text pattern | tx | O, GR | web, tui | — | +| conv | "Stopped" marker after Stop until an aborted response shows | client flag | eph | O, GW | web | stop | +| conv | Tool paths relative to the conversation's own cwd; open a file from a card in the workspace | cwd | eph | O, GW | web | workspace | +| conv | Expand tools / show thinking toggles; reading position kept per conversation across switches and reloads | client state | rst (TUI snapshot) | O, GR | web, tui | — | +| conv | Follow output, scroll-to-bottom button, keep the reader's position while sending | client state | eph | O, GR | web, tui | — | +| conv | Read-only bar for view-only guests ("Joined as X (unverified)"); share-ended screen | grant name | eph | GR | web, tui | access | +| conv | "Session unavailable" with Retry and Show logs (owner), "ask the owner" (guest) | worker error | rst | O, GR | web | worker status, logs | +| conv | Leave the view with a notice if the session is archived or removed elsewhere | catalog row | eph | O, GR | web | session list | + +## Sending and the queue + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| conv | Send a message: text ≤1 MB, at least text or one attachment | text | tx | O, GW | web, tui→wkr | admission rules | +| conv | Retried send with the same request id is admitted once, per author (id ≤200) | request id | tx (Durable) | S | wkr | — | +| conv | While busy: steer the current run (default) or queue a follow-up | mode | tx | O, GW | wkr | — | +| conv | Wait before admitting input until prompt resources are loaded (first message of a fresh worker) | — | eph | S | wkr | prompt resources | +| conv | Attribute input: speaker block in the model message, structured author in the entry; owner name read at write time | author | tx | S | wkr | settings | +| conv | Legacy speaker-block parsing of old entries (drop? p3 starts empty) | — | tx | S | all | — | +| conv | Expand `/template` and `/skill:` on the worker before sending | text | eph | O, GW | wkr | prompt resources | +| conv | `@` and Tab file completion | file index | eph | O, GW | web, tui | file index | +| conv | List queued inputs with their mode | inbox | tx | O, GW | web, tui | view | +| conv | Queue item back to editor: caller's text returns; other authors' texts become their restored drafts | texts | tx + drafts | O, GW | wkr | restored drafts | +| conv | Remove a queue item (discarded, whoever wrote it) | — | tx | O, GW | wkr | — | +| conv | Stop: withdraw every queued input, abort the run; own texts to editor, others' to their restored drafts; allowed even when read-only | texts | tx + drafts | O, GW | wkr | restored drafts | +| conv | Withdraw own queued inputs (Alt+Up), report how many were already placed | texts | tx | O, GW | wkr, tui | — | +| drafts | Restored drafts per (conversation, author); claimed once, first tab wins | ~KB each | rst (should be pers) | O, GW (own) | wkr→web, tui | stop, dequeue | +| drafts | Composer draft survives navigation: web one draft in sessionStorage; TUI per conversation plus across worker swaps | text | eph / rst | O, GW | web, tui | — | +| drafts | TUI pending send ids reused, sent once connected ("Sending when connected…") | request id | rst | O, GW | tui | reconnect | +| drafts | TUI drafts and snapshots partitioned per scope, pending sends never replayed into another scope | snapshot | rst | O, GW | tui | project plugins | + +## Configuration and conversation actions + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| conv | Choose model and thinking level: owner only, known model, known level | ref | tx | O | wkr | model catalog | +| conv | The root conversation's model also updates the session's default (sidebar, next worker start) | ref | pers | S | srv | session list | +| conv | Compact now (instructions ≤10k); abort running compactions (Escape) | task id | tx | O, GW | wkr | — | +| conv | Start a new context with a handoff (≤100k); owner only | text | tx | O | wkr | — | +| conv | Read-only conversations (scripts' one-shot agents): refuse send, dequeue, configure, compact, shell; Stop allowed; set at creation | flag | tx | S | wkr | subagents | +| conv | Fork at a person's message into a new session, the message waiting in its editor (root conversation only) | — | pers | O | tui, srv | session clone | +| conv | Clone the whole session | — | pers | O | srv | session clone | +| conv | Copy the last answer (TUI via launcher confirm, ≤1 MiB; web copy button) | text | eph | O, GR | web, tui | launcher | +| conv | External editor (Ctrl+G); double Ctrl+C exits; double Escape opens fork | text | eph | O, GW | tui | launcher | +| conv | Open subagent and one-shot conversations; subagent bar shows tools, cwd, archive/restore | child list | pers | O, GW (archive) | web, tui | subagents | +| conv | New-session screen: folder, env, model, first message with attachments | — | pers | O | web | sessions, uploads | +| conv | Inline images ≤5 MiB if the model accepts images, otherwise list the path | image bytes | tx | S | wkr | uploads, model metadata | + +## Shell commands + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| conv | `!cmd` (output joins context) and `!!cmd` (display only): one per conversation, in the conversation's cwd on its environment | command ≤100k | tx (entry) | O, GW | wkr, env | execution env | +| conv | Live output tail: 50 KB / 2000 lines, sampled every 200 ms, at most 100 KiB/s; full output spilled to a file | ≤50 KB live | eph | O, GR | wkr→web, tui | — | +| conv | Shell result becomes an entry, placed at the run's next boundary while busy; never starts a run | entry | tx | S | wkr | — | +| conv | Cancel a command: its author or the owner; timeout reported | — | eph | O, GW | wkr | — | +| conv | Retried shell request runs once (100 keys kept) | ids | rst | S | wkr | — | +| conv | Shell cancelled when the conversation has no viewer for 30 s, or on reload (drop? accidental) | — | — | S | wkr | — | + +## Chat mode + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| chatmode | Switch per conversation: owner switches, everyone sees it; forks and new subagents start off | bool | tx-side doc | O / GR | wkr | — | +| chatmode | Messages without `@pi` become display-only chat entries (author, text, attachments, reply) | entry | tx | O, GW | wkr | send | +| chatmode | `@pi` mention: digest of the chat since the last mention, recording which entries it passed on | ≤100 messages, ≤20 files, ≤4 images; scan ≤2000 | tx | O, GW | wkr | send, uploads | +| chatmode | Composer chip with the count of chat messages not yet passed on | count | eph | O, GR | web, tui | view | +| chatmode | Reply to a person's or Pi's message: client names the entry, worker derives author and a 60-char quote | quote | tx | O, GW | web, tui, wkr | — | +| chatmode | Quote line and "with N chat messages" note in the transcript; dashed bubble; `@pi` completion first; hint text | — | — | O, GR | web, tui | renderers | + +## Cross-session messages + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| xconv | Agent lists other top-level sessions: id, title, model, working/idle/stopped; archived excluded; ≤200 | ~20 KB | eph | W (root only) | srv | session list, worker status | +| xconv | Agent sends another session an attributed message (≤32 KiB, steer or follow-up); starts the recipient's worker | text | tx (recipient) | W | srv→wkr | start worker | +| xconv | Send is idempotent by tool call id; retried once if the recipient's worker went away | ids | tx | S | srv, wkr | — | +| xconv | Agent reads another session's latest completed answer without sending (≤64 KiB, scans ≤2000, busy flag) | text | eph | W | wkr | — | +| xconv | Only owner-initiated turns or authenticated peer messages may use it (follows nested calls ≤32 deep, scans ≤2000); guests never | — | — | S | wkr | attribution | +| xconv | Refuse a stale worker generation, sending to yourself, archived sender or recipient; recheck after start | — | — | S | srv | worker hosts | +| xconv | System prompt section describing the tool | text | — | S | wkr | prompt | +| xconv | Peer input shown with sender label; owner opens the peer session, restoring it if archived | — | tx | O (link), GR (label) | web | session list | +| xconv | Sender attribution as a JSON text prefix (drop? use a structured author) | — | tx | S | — | — | + +## Uploads (attachments) + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| files | Upload an attachment: ≤25 MiB, same origin, outside the project directory | blob ≤25 MiB | pers | O, GW | web→srv | — | +| files | At most 20 per message; ids resolved only to this session's uploads | ids | pers | S | wkr | — | +| files | Copy attachments to a remote environment before admission; no fallback on failure | blob | pers (env) | S | wkr, env | execution env | +| files | TUI attach through the launcher (picker, path confirm, Ctrl+V); remove last or all | upload id | rst | O, GW | tui | launcher | + +## Sent files + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| sent | Agent offers a file: private snapshot ≤25 MiB read through the environment (remote too), optional name | blob | pers | W | wkr, env→data dir | execution env | +| sent | Download for viewers: cross-site refused, byte ranges for audio/video, safe filename | stream | pers | O, GR | srv | session list | +| sent | Card in the transcript; viewer panel with preview (image, audio, video, text) | — | tx | O, GR | web | — | +| sent | TUI list of files sent in the loaded transcript; text preview ≤256 KiB (binary detected); save or open in browser | ≤256 KiB | eph | O, GR | tui, wkr | launcher | +| sent | Workspace also creates download snapshots | blob | pers | O | srv | workspace | + +## Artifacts + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| artif | Agent creates or updates an artifact: html, svg, markdown or text; ≤2 MiB; slug name; title ≤200; new immutable numbered version each time | ≤2 MiB per version | pers | W | wkr→data dir | — | +| artif | Per-conversation index, not rewound by compaction or reset; forks start empty | ~100 B per version | pers | S | wkr | — | +| artif | List: count button, picker newest first | list | pers | O, GR | web, tui | index | +| artif | View: HTML/SVG in a sandboxed frame with CSP (https CDNs allowed, no app identity); markdown rendered; text; download; open tab | ≤2 MiB | pers | O, GR | srv, web | — | +| artif | Preview diagnostics bridge: console errors, CSP violations, WebGL loss, ≤50 kept | messages | eph | O, GR | web | — | +| artif | TUI card (title, type, version, text content when expanded); inline view ≤512 KiB (HTML/SVG as source); open in browser; download | ≤512 KiB | eph | O, GR | tui, wkr | launcher | + +## Session lifecycle and liveness + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| lifecycle | Clone copies uploads, downloads and artifacts | blobs | pers | O | srv | clone | +| lifecycle | Delete or archive cleans up blobs (missing today) | — | — | S | srv | session delete | +| presence | Worker stays up while a person views; running calls block idle stop; sends refused during a move | counters | eph | S | wkr | keep-alive | +| presence | Viewer count in the footer | count | eph | O, GR | web | presence | + +## (1) Cross-cutting needs + +| need | where it shows | +|---|---| +| Blob storage with caps, immutable ids, private modes, range reads | uploads, sent files, artifacts, shell spill files | +| Per-session data that clone copies and delete removes | blobs, restored drafts, chat-mode switch, artifact index | +| Transcript-bound vs. app state | chat-mode switch and artifact index have "fork starts empty" and "not rewound" semantics | +| Readers that must not start a worker | artifact list and preview, sent-file preview, cost sheet, peer state list | +| Long work outliving the call, with cancel, dedupe and display until placed | shell, peer send | +| Cancellation | Stop, abort compaction, abort shell, caller abort, worker going away | +| Keep-alive vs. presence | viewers vs. lingering tabs vs. active calls vs. tasks | +| Idempotency per author request id | send, write, shell, peer send | +| Authorization split | view / send / owner-only, applied in every route and method | +| Launcher grants for TUI side effects | copy, attach, download, open in browser, external editor | +| Bounded scans | history, mention digest, attribution, latest answer: each ≤2000 entries | +| Remote-environment copies | attachments, sent files | + +## (2) Functions duplicated today + +| function | duplicated in | +|---|---| +| Input gate loop | send, write, shell, peer send | +| Latest answer text | TUI copy, web copy, peer answer scan | +| Peer sender parsing | transcript rendering, tool attribution check | +| Artifact list | client-side from the transcript, and the worker index | +| Download snapshot writers | send_file, workspace | +| Size and ID validation | uploads, downloads, artifacts each have their own | +| Restoring texts to the editor | web and TUI each implement it | +| Per-conversation state polling via commit signal plus loop | chat-mode, artifacts | +| Transcript building and rendering | web and TUI each have their own renderers (shared builder) | +| Preview limits | 256 KiB sent files vs. 512 KiB artifacts | diff --git a/apps/p2/docs/p3-functions/core.md b/apps/p2/docs/p3-functions/core.md new file mode 100644 index 000000000..f461d8b9c --- /dev/null +++ b/apps/p2/docs/p3-functions/core.md @@ -0,0 +1,199 @@ +# Core runtime, hosts and UI shells + +| domain | function (plain words) | data it needs or produces (what, roughly how big) | durability | who may see or do it | where it must be available | depends on (other functions) | +|---|---|---|---|---|---|---| +| sessions | Browse sessions grouped by execution machine and directory | Session summaries; titles, locations, timestamps, pin/archive flags | persisted metadata; ephemeral grouping | owner; guests see their shared session | web; server supplies summaries | session catalog, access, worker status | +| sessions | Put pinned sessions first within a directory | One boolean per session | persisted | owner | web; server | session catalog | +| sessions | Rename a session through its options sheet | Nonempty title | persisted | owner | web; server | session catalog | +| sessions | Archive a session through its options sheet | Session identity | persisted | owner | web; server | archive operation, worker stop, access revocation | +| sessions | Restore an archived session when opening it | Session identity; optional child conversation | persisted | owner | web; server | restore operation, navigation | +| sessions | Browse child conversations and archived children | Child IDs, names, models, activity, archive flags | persisted metadata; ephemeral expansion | owner, guest read, guest write within shared session | web | conversation listing, access | +| sessions | Remember choices for the next new session | One location and model/thinking choice | until restart of page | owner | web | environment selection, model selection | +| sessions | Show work, unread answers and worker failures in session lists | Busy conversation IDs, unread flag, error string | ephemeral display; read marks supplied by session storage | owner; scoped status for guests | web, TUI | activity reporting, read-mark tracking, worker lifecycle | +| sessions | Pause a worker safely for cloning | Session ID, admission fence, prior running state | ephemeral coordination | system for owner operation | server, session worker | task inspection, worker stop, external database/blob copying | +| sessions | Stop a worker before archive/delete cleanup | Session ID; lifecycle operation queue | ephemeral coordination | system for owner operation | server, session worker | worker shutdown; catalog/blob cleanup is outside core | +| conversations and agent | Open a session’s durable conversation store | One SQLite database per session; size grows with history and work | transcript-bound | worker, system | session worker | exclusive process lock, model configuration, execution environment | +| conversations and agent | Initialize the root conversation | Model, thinking level, cwd | transcript-bound | worker, system | session worker | session catalog, durable storage | +| conversations and agent | Reject startup when the saved root model is unavailable | Model identity and configured model inventory | until restart; failure remains until retry | owner sees failure; worker validates | session worker, server | model configuration, worker error reporting | +| conversations and agent | Resume interrupted durable work after worker restart | Persisted tasks and submissions | transcript-bound | worker, system | session worker | durable storage, installed tools and prompt resources | +| conversations and agent | Open an archived session for indexing without resuming work | Startup flag; archived transcript | ephemeral mode; transcript-bound input | system | server, session worker | worker startup, search rebuilding | +| conversations and agent | Wait for initial prompt resources before accepting input | Readiness/cancellation state | ephemeral | worker, system | session worker | prompt-resource loading | +| conversations and agent | Keep tool definitions consistent throughout an executing phase | Immutable definition snapshot | until phase ends | worker, system | session worker | tool availability, reload coordination | +| conversations and agent | Enforce conversation-specific human-input restrictions | Read-only reason; conversation ID | ephemeral policy | worker enforces; owner/guest write affected | session worker | conversation ownership; stop remains separately allowed | +| conversations and agent | Attribute human input independently of typed text | Author kind, stable ID, display name, text, attachment IDs/names | transcript-bound | owner, guest read, guest write; worker assigns | session worker, web, TUI | authenticated principal, owner-name setting | +| conversations and agent | Put authoritative speaker identity before model-visible input | One short structured speaker block | transcript-bound | worker produces; model consumes | session worker | author attribution | +| conversations and agent | Read legacy speaker and attachment encoding **(drop? old-session compatibility only)** | Leading speaker block and trailing attachment note | transcript-bound | worker, owner, scoped guests | session worker, web, TUI | legacy transcript | +| conversations and agent | Make unanswered model failures visible after crashes and reconnects | Display-only error entry and deduplication receipt per failed input; recovery scans 100 records/page | transcript-bound | worker writes; session viewers read | session worker, web, TUI | submission settlement, durable commits | +| conversations and agent | Build a readable transcript from committed and live activity | Entries, streaming response, tool results, nested calls, pending inputs, reset/compaction markers | ephemeral projection of transcript-bound data | owner, guest read, guest write | web, TUI; worker supplies data | conversation snapshots, retained history | +| conversations and agent | Preserve visible history when context resets or compacts | Loaded entries merged by entry ID | until restart of client; bounded number of retained conversations, not total history bytes | owner, scoped guests | web, TUI | history paging | +| conversations and agent | Load older transcript pages without accepting stale replies | Paging cursor, snapshot epoch, loaded entries | ephemeral | owner, scoped guests | web, TUI, session worker | history retrieval | +| conversations and agent | Keep recently visited conversations inexpensive to reopen | Four inactive history stores; conversation references linger two minutes, cap four | until restart; timed retention | owner, scoped guests | web, TUI | subscription lifetime, history cache | +| conversations and agent | Reveal a searched or quoted message, loading history if necessary | Session, conversation and entry IDs | ephemeral | owner, scoped guests | web | navigation, history paging | +| conversations and agent | Show thinking, streamed answers and tool progress | Potentially large text; expandable tool/thinking cards | ephemeral display; transcript-bound source | owner, scoped guests | web, TUI | transcript projection | +| conversations and agent | Show tool-specific results and nested script calls | Commands, paths, diffs, files, artifacts, child-agent links, nested call statuses | ephemeral display; transcript-bound source | owner, scoped guests | web, TUI | transcript projection, file/artifact/conversation navigation | +| conversations and agent | Estimate current context usage | Active entries with edits applied; token estimate and model limit | ephemeral derived values | owner, scoped guests | session worker, web, TUI | active transcript, model metadata | +| conversations and agent | Show latest-response cache effectiveness | Input/cache-read/cache-write token counts | transcript-bound source; ephemeral calculation | owner, scoped guests | web, TUI | usage accounting | +| conversations and agent | Inspect response costs separately from session totals | One cost/model/time/result row per response; charts | transcript-bound source; ephemeral display | admitted conversation viewers | web; session worker supplies rows | usage accounting, active transcript | +| conversations and agent | Record a user-run shell command without starting an agent turn | Author, command, exit/cancel/error state, output tail ≤2,000 lines/50 KiB, optional spill path | transcript-bound; spill file separately stored in execution environment | owner, guest write; scoped viewers read result | session worker, remote env, web, TUI | shell execution, durable write | +| conversations and agent | Choose whether shell output enters the next model context | Inclusion flag: `!` includes, `!!` display-only | transcript-bound | owner, guest write | session worker, web, TUI | shell-result recording | +| composition | Keep an unsent web draft through navigation | App-scoped text, send mode, reply target, submission identity, errors | until restart; text also persisted in tab storage | owner, guest write | web | current conversation, send operation | +| composition | Restore the matching web draft after page reload | One saved conversation key and draft string | persisted in tab storage | owner, guest write | web | browser storage | +| composition | Retain pending attachments across web navigation | Native files, upload IDs, preview URLs; ≤25 MiB/file | until restart of page | owner, guest write | web | file selection, upload | +| composition | Show an image preview before upload | Browser object URL and original image | until restart of page | owner, guest write | web | attachment selection | +| composition | Keep a reply target scoped to its conversation | Entry ID and display name | ephemeral; cleared on conversation change | owner, guest write where replies enabled | web | transcript selection, reply support | +| composition | Retain stop/restored-draft state without attaching it to another conversation | Stopped conversation identity; restored text supplied by send/stop implementation | ephemeral; restored text can enter saved draft | owner, guest write | web, TUI | stop/dequeue and draft restoration outside core | +| composition | Keep TUI drafts and uncertain submissions per conversation | Text; request ID and steer/follow-up mode per session/conversation | until restart; carried across UI-worker replacement | owner, guest write | TUI | send acknowledgements, snapshot transfer | +| composition | Expand prompt commands before sending | Draft text and expanded template/skill text | ephemeral; sent result becomes transcript-bound | owner, guest write | web, TUI, session worker | prompt resources | +| composition | Complete cwd-relative file references | Query ≤200 characters; ≤100 requested results, truncation flag | ephemeral | owner, guest write; not guest read | web, TUI, session worker, remote env | confined file index, filesystem changes | +| composition | Insert quoted path completions correctly | Cursor location, token prefix, file/directory suggestion | ephemeral | owner, guest write | web, TUI | file completion | +| composition | Keep agent mentions and slash commands distinct from file completion | Short input token; `@pi` and slash-command context | ephemeral | owner, guest write | web, TUI | command completion | +| environments and execution | Select local or remote execution for a session | Environment name, cwd, connection | persisted location; until restart connection | owner configures; worker uses | server, session worker, remote env | environment configuration | +| environments and execution | Use a local execution daemon when available | Daemon path and lazy connection; otherwise direct Node execution | until restart | worker, system | session worker | local execution support | +| environments and execution | Resolve each conversation’s own working directory | Per-conversation cwd and environment cache | transcript-bound cwd; until restart cache | worker, system | session worker, remote env | conversation configuration, session location | +| environments and execution | Repair cwd-following conversations before work resumes | Root and follower IDs; changed cwd values committed together | transcript-bound | worker, system | session worker | conversation listing, durable commits | +| environments and execution | Refuse relocation while work cannot safely move | Active calls, tasks, pending submissions, busy conversations, refusal reason | ephemeral | owner requests; worker/system enforce | server, session worker | liveness, durable inspection, relocation checks | +| environments and execution | Move an idle running session without losing client views | New machine/cwd; location revision; switched environment connections | persisted catalog and conversation cwd; ephemeral connections | owner; system coordinates | server, session worker, remote env | admission fence, catalog update, environment opening | +| environments and execution | Recover a failed or hung relocation | Committed target location; 60-second move timeout | persisted target; ephemeral recovery | system; owner sees failure | server, session worker | worker restart, current catalog | +| environments and execution | Reopen location-dependent resources after a move | Location revision and notifications | ephemeral | worker, system | session worker, remote env | committed relocation, prompt/workspace watches | +| background and long-running work | Start a session worker on demand | Session initialization, selected capabilities, generation, PID | until restart | system for authorized session access | server, session worker | session authorization, catalog, exclusive lock | +| background and long-running work | Prevent duplicate workers opening the same store | Lock keyed by real data directory and session ID | ephemeral OS-held lock | system | server, session worker | process locking | +| background and long-running work | Bound worker startup | Readiness state; 60-second timeout | ephemeral | system; owner sees error | server, session worker | worker bootstrap | +| background and long-running work | Stop workers only after genuine inactivity | Viewer, active-call and durable-task counts; default five-minute idle interval | ephemeral | system | server, session worker | liveness reporting | +| background and long-running work | Keep detached active work from appearing idle | Active-call lease for work outside ordinary request lifetime | ephemeral | worker, system | session worker | work cancellation/completion | +| background and long-running work | Distinguish shutdown from capability removal | Stopping flag; interrupted-work state | ephemeral flag; transcript-bound work | worker, system | session worker | durable resumption, reload disposal | +| background and long-running work | Report worker crashes without corrupting newer status | Generation, exit code/signal, redacted stderr tail ≤4,000 characters | until restart | owner; system produces | server | worker lifecycle, log redaction | +| background and long-running work | Retry a failed worker explicitly | Session ID; clears sticky startup failure and related notices | until restart | owner | server, web, TUI | worker stop, fresh access | +| background and long-running work | Shut down workers on server loss or termination | Shutdown signal; three-second forced-exit fallback | ephemeral; durable work remains persisted | system | server, session worker | storage close, execution cleanup | +| access | Identify the owner, scoped guest or worker behind a request | Role, device ID or guest ID/session/mode/revision; worker generation | ephemeral authenticated identity; credentials elsewhere persisted | system assigns | server, session worker, web, TUI | credential authentication, transport provenance | +| access | Admit only known local or trusted tunnel origins | Loopback address/port; allowed HTTPS hosts; forwarded-header checks | until restart configuration | system | server | listener configuration | +| access | Reject cross-origin browser control and framing | Origin and Fetch Metadata; nonce-based CSP; trusted relay-frame exception | ephemeral per request | system | server, web | request provenance | +| access | Preserve local-owner and guest-cookie distinctions | Per-port loopback-owner cookie; existing owner/guest credentials | persisted cookies | owner, guest read, guest write | server, web | credential authentication | +| access | Authorize access to a particular session before starting its worker | Session ID and current guest grant | ephemeral decision | owner, session-scoped guests; system | server | grant validity | +| access | Separate reading from sending, stopping and uploading | Guest `view`/`full` mode | persisted grant; ephemeral checks | owner, guest read, guest write | server, session worker, web, TUI | authenticated identity | +| access | Revoke live access, not just future sign-ins | Device ID, guest ID or session; grant revision | persisted revocation externally; immediate ephemeral teardown | owner/system | server, session worker, web, TUI | live connection tracking | +| access | Show guest names as unverified identities | Guest name ≤40 characters; read/write label | persisted grant; ephemeral display | owner, guest read, guest write | web, TUI | authenticated identity | +| remote access | Redeem an owner link in the TUI | Origin-bound token → owner cookie; ten-second request timeout | persisted launcher credential | owner | TUI, server | owner-link service | +| remote access | Redeem a share link in the TUI | Origin-bound token, sanitized name, existing guest cookie → session/cookie; response ≤2 KiB | persisted launcher credential | guest read, guest write | TUI, server | share service | +| remote access | Keep owner and guest credentials separate per server | Origin-keyed cookie records; private directory `0700`, file `0600` | persisted | local TUI user; never downloaded UI code | TUI | local configuration | +| remote access | End reconnect loops when access is revoked | HTTP 401/403 or admitted-session probe | ephemeral | owner, guest read, guest write | web, TUI, server | authentication checks | +| remote access | Forget revoked TUI credentials and offer a fresh link | Credential kind and origin; replacement link | persisted credential removal | local TUI user | TUI | access-ended detection | +| remote access | Sign out one or all owner devices from the TUI | Confirmed scope; credential deletion | persisted | owner with launcher confirmation | TUI, server | device revocation | +| links and transport | Carry authorized calls and live state between clients and workers | JSON/state frames; server input ≤1 MiB, 32 pending calls/connection | ephemeral | owner, scoped guests, worker, system | server, session worker, web, TUI | authentication, session routing | +| links and transport | Disconnect slow or oversized clients without killing the process | Output backlog budget 4 MiB beyond largest unfinished frame | ephemeral | system | server | socket accounting | +| links and transport | Reconnect and rebuild client subscriptions from snapshots | Connection state and held interests; exponential backoff from 250 ms, capped at five seconds | ephemeral | owner, scoped guests | web, TUI | live transport | +| links and transport | Avoid replaying stale TUI connection traffic | Frame/queue budget 4 MiB; ten-second handshake; no redirects | ephemeral | system | TUI | connection replacement | +| links and transport | Refresh an outdated browser protocol implementation | Local and server protocol versions | ephemeral | system | web | reconnect handshake, page reload | +| models and settings | Validate and save agent defaults and execution policy | Small JSON: model/thinking, streaming/retry/cache, compaction, tool concurrency, queue modes, codemode, owner name | persisted | owner; worker/system read | server, session worker | settings parsing, coordinated file writes | +| models and settings | Configure an ordered model-cycling scope | ≤500 patterns, each ≤300 characters; optional thinking suffix | persisted | owner | server, TUI | model inventory, settings | +| models and settings | Resolve environment-provided default model | Provider/model pair; rejects half-configured pair | until restart | owner/system | server | startup configuration | +| models and settings | Accept old default-model environment aliases **(drop? migration compatibility only)** | Two legacy environment names | until restart | owner/system | server | default-model resolution | +| models and settings | Search models available through signed-in providers | Model/provider names, IDs, context limits, thinking options; web shows ≤300 matches | ephemeral display of configuration | owner where model controls exposed | web, TUI | model inventory, credential-status summaries | +| models and settings | Choose thinking level supported by the selected model | One enum value; unsupported previous choice resets to off | transcript-bound conversation setting | owner or permitted conversation configurator | web, TUI, session worker | model metadata, configure operation | +| models and settings | Save the selected model as the new-session default | Model identity and thinking level | persisted | owner | web, TUI, server | settings write | +| models and settings | Observe configuration and prompt files without losing changes during reads | Watched paths, generation counters, last valid state; invalid-save retry after 200 ms | until restart cache; files persisted | system | server, session worker, remote env | file watches, fallback polling | +| models and settings | Serialize private configuration writes | Lock, current file, validated replacement; 30-second stale/deadline window | persisted | owner/system | server | private filesystem, cancellation | +| models and settings | Keep provider HTTP failures from wedging an entire origin | HTTP/1.1 connection pool; fetch/body errors | ephemeral | system | server, session worker | outbound network | +| session files and blobs | Display attachment and generated-file cards | File ID/name/size, authorized URL; small metadata | transcript-bound references | owner, guest read, guest write | web, TUI | file storage and authorization | +| session files and blobs | Preview safe stored-file types | Text ≤1 MiB by declared size; raster image/audio/video URLs | ephemeral preview; persisted file | owner, scoped guests | web | authorized downloads, conservative type allowlist | +| session files and blobs | Open versioned artifacts explicitly | Artifact name/version/type; view/download URLs | persisted artifact; ephemeral viewer state | owner, scoped guests | web, TUI | artifact service; executable preview isolation supplied there | +| session files and blobs | Sanitize display/download filenames | Basename stripped of path/control syntax; bounded strings | ephemeral derived metadata | system | server, web, TUI | uploaded/generated filename | +| workspace | Browse the viewed conversation’s working tree | Directory entries, sizes, ignored/link markers; UI reports 2,000-entry listing cap | ephemeral | owner, guest write; not guest read | web, session worker, remote env | workspace authorization, confined filesystem reads | +| workspace | Find a file by fuzzy name | Indexed paths; query ≤200 characters; top 200 displayed; UI reports 50,000-file index cap | ephemeral | owner, guest write | web, session worker, remote env | file listing/index | +| workspace | Search file contents with include/exclude filters | Literal query ≤200 characters; each glob field ≤500; bounded matches | ephemeral | owner, guest write | web, session worker, remote env | workspace search | +| workspace | Inspect working-tree changes | Status letters, renamed paths, grouped change tree | ephemeral | owner, guest write | web, session worker, remote env | repository status | +| workspace | Browse commit history | Commit summaries, upstream/unpushed markers; 30/page, refresh ≤500 | ephemeral projection of repository history | owner, guest write | web, session worker, remote env | repository queries | +| workspace | Inspect a file or diff at a selected commit | Commit hash, path, text/diff; historical media download not offered by this shell | ephemeral projection of repository history | owner, guest write | web, session worker, remote env | repository file/diff queries | +| workspace | Switch among file, diff and preview views | Path, line, view, context-line count, wrap preference | ephemeral route; persisted wrap preference | owner, guest write | web | file reads, preview policy | +| workspace | Upload files into the selected working directory | Selected files, target folder, progress/error | persisted filesystem changes | owner, guest write | web, server, session worker, remote env | authorized workspace upload | +| workspace | Draft line-linked review comments | Path, line/range, quote, text, view and optional commit | persisted in browser local storage | owner, guest write | web | workspace file/diff display | +| workspace | Preserve an unfinished review-comment editor | Editor location and in-progress text | persisted in browser local storage | owner, guest write | web | review comments | +| workspace | Send selected review comments as a conversation message | Sorted comments; remove only successfully sent draft IDs | transcript-bound after send; local drafts until then | owner, guest write | web, session worker | message submission | +| workspace | Refresh visible workspace data without flashing away old results | Watch revision; 500-ms debounce; 300 cached query answers | until restart of page | owner, guest write | web | filesystem changes, query cache | +| workspace | Preserve workspace navigation and layout | Tab, folder/file/commit/search, expanded folders, panel width and scroll positions | ephemeral expansion; persisted route/layout portions | owner, guest write | web | navigation, browser storage | +| terminals | Expose the session terminal panel without granting it to guests | Terminal identity and start intent | ephemeral route | owner only | web; server/session machine provide terminal | terminal service, session location | +| UI shells | Serve authenticated installable web pages and content-named assets | HTML, manifest, icons, JS/CSS/assets; UUID session and positive conversation routes | until restart bundles; browser-cached immutable files | owner, scoped guests | server, web | access gate, UI build | +| UI shells | Provide themed sign-in, join and access-ended pages | Small server-rendered HTML; escaped titles; nonce-protected scripts | ephemeral | owner, guest read, guest write, unauthenticated visitor as appropriate | server, web | access flow | +| UI shells | Adapt navigation to desktop, mobile Back and embedded browsers | Conversation route, drawer/sheet/viewer stack, workspace/terminal route | persisted in tab history/storage | owner, scoped guests | web | browser navigation facilities | +| UI shells | Restore per-step scroll positions | Up to 200 saved positions; content arrival wait ≤2 seconds | persisted in tab storage | owner, scoped guests | web | navigation, asynchronous content | +| UI shells | Follow new answers without moving someone reading older text | Bottom-follow flag and visible-element anchors | ephemeral | owner, scoped guests | web, TUI | transcript updates, layout measurement | +| UI shells | Preserve transcript reading position across TUI replacements | Entry/item key and line offset per conversation | until restart; included in replacement snapshot | owner, scoped guests | TUI | history loading, snapshot transfer | +| UI shells | Render rich text without executing transcript HTML | Markdown, tables, code/diff blocks, permitted inline formatting and validated links | ephemeral display | owner, scoped guests | web, TUI | transcript text, syntax rendering | +| UI shells | Render diagrams according to user display policy | Mermaid source; off/final/streaming preference | ephemeral display; persisted preference | owner, scoped guests | TUI | Markdown transformation | +| UI shells | Remember browser theme and sidebar state | System/light/dark choice; collapsed flag | persisted in browser local storage | owner, scoped guests | web | browser storage/theme | +| UI shells | Discover and invoke context-appropriate TUI commands | Command IDs, slash names, shortcuts, role/screen/focus conditions | until restart; configurable bindings persisted | owner, scoped guests according to command | TUI | current access and screen | +| UI shells | Explain and report shortcut conflicts | Resolved bindings, conflicting contexts, unknown IDs, source labels | ephemeral | local TUI user | TUI | command resolution, preference validation | +| UI shells | Customize local TUI appearance and interaction | Whitelisted settings; preference file ≤256 KiB; ≤64 custom themes, ≤1 MiB combined | persisted | local TUI user | TUI | trusted local-file reads, validation | +| UI shells | Merge server display defaults below local preferences | Built-in → server → local pi files → p2 preferences; privileged keys excluded from server control | persisted sources; ephemeral merged state | owner supplies server defaults; each visitor controls local settings | server, TUI | preference validation | +| UI shells | Hot-reload local TUI preferences | Changed preference/theme files; 100-ms debounce; ≤50 diagnostics | until restart derived state; source files persisted | local TUI user | TUI | filesystem watches | +| UI shells | Save UI-requested local preferences only after showing the diff | Validated patch; excludes launcher-owned privilege settings | persisted | local TUI user confirms | TUI launcher | trusted confirmation, current preference file | +| UI shells | Replace downloaded TUI code while preserving selected UI state | Hash-verified bundle ≤8 MiB; manifest ≤8 KiB; snapshot ≤1 MiB | until restart; snapshot survives replacement only | system; visitor can request reload | server, TUI | bundle publication, sandbox startup, snapshot validation | +| UI shells | Read old TUI snapshot formats **(drop? replacement-version compatibility only)** | Legacy session-keyed drafts and screen names | ephemeral migration | system | TUI | snapshot restoration | +| TUI local safety | Keep downloaded UI code away from local files, network and credentials | Permission-dropped worker; empty environment; narrow message bridge | ephemeral | system enforces for every visitor | TUI launcher | supported Node runtime | +| TUI local safety | Bound downloaded UI memory and startup | Heap limits 96/16 MiB; code range 64 MiB; stack 4 MiB; process RSS watchdog 512 MiB; startup 10 seconds | ephemeral | system | TUI launcher | sandbox lifecycle | +| TUI local safety | Filter terminal output before it reaches the real terminal | Complete frames ≤64 KiB; limited cursor/style/mode controls; OSC/DCS discarded | ephemeral | system | TUI launcher | output parser, local mouse policy | +| TUI local safety | Authorize one privileged local action from a real user gesture or confirmation | Action, origin, UI-worker identity, session/conversation, cancellation signal; ten-minute grant lifetime | ephemeral | local TUI user | TUI launcher | trusted input and confirmation | +| TUI local safety | Cancel local-action authority after navigation or UI replacement | Grant IDs and target identity | ephemeral | system | TUI launcher | navigation, sandbox lifecycle | +| TUI local safety | Copy text using trusted local clipboard access | Selected/last-answer text ≤1 MiB; selection gesture window two seconds | ephemeral; external clipboard persists | local TUI user | TUI launcher | user key, selection gesture or explicit confirmation | +| TUI local safety | Upload a locally chosen file without trusting server-supplied file facts | Verified regular-file descriptor; no final symlink; ≤25 MiB | persisted server upload; ephemeral local grant | owner, guest write with local consent | TUI launcher, server | file picker, upload authorization | +| TUI local safety | Paste clipboard files, images or text safely | Up to ten files; confirmed image upload; sanitized bracketed text paste | ephemeral or persisted upload | local TUI user; upload requires write access | TUI launcher, server | clipboard access, attachment upload | +| TUI local safety | Download a file without overwriting local files | Authorized session file/artifact; ≤25 MiB; 60-second timeout; safe unique filename | persisted local file | admitted visitor with local confirmation | TUI launcher, server | download authorization, bounded streaming, cancellation cleanup | +| TUI local safety | Open a browser URL without forwarding TUI credentials | Confirmed HTTPS external URL or allowlisted same-origin resource | ephemeral | local TUI user | TUI launcher | URL validation, trusted confirmation | +| TUI local safety | Edit draft text in the visitor’s external editor | Text ≤1 MiB; private temporary file; result tied to original grant | ephemeral; temporary file removed | local TUI user | TUI launcher | local editor preference, terminal handoff | +| TUI local safety | Suspend and resume the TUI process cleanly | Terminal modes, process job-control state | ephemeral | local TUI user | TUI launcher | terminal handoff; unavailable on Windows | +| TUI local safety | Restore terminal state after overlays, external programs, reloads and crashes | Mode state, title/progress/status, dimensions clamped to 500×200 | ephemeral | system | TUI launcher | terminal lifecycle | +| dev and reload | Detect source changes without applying them automatically | File hashes, loaded dependencies, bundle inputs; dirty status | until restart | owner, system | server, session worker, web, TUI | source watching | +| dev and reload | Distinguish reloadable changes from runtime restart requirements | Changed source paths and loaded baselines | until restart | owner, system | server | dependency/input tracking | +| dev and reload | Apply explicitly selected capability changes across running processes | Candidate selection; readiness/refusal report; previous/current code versions | until restart | owner; authorized local automation | server, session workers | compatible reload, dependency checks | +| dev and reload | Refuse removing functionality still in use | Provider/consumer use records across server and workers | ephemeral | system; owner sees reasons | server, session workers | active-use tracking | +| dev and reload | Publish UI code only after successful backend change | Prepared web/TUI bundles; each ≤8 MiB excluding web source maps | until restart | system | server | successful reload | +| dev and reload | Let just-served old web pages finish loading during a reload | At most two previous bundles retained for 60 seconds | ephemeral | owner, admitted guests | server, web | content-named assets | +| dev and reload | Detect changed web code and reload the page | Manifest version; five-second visible-page polling plus visibility check | ephemeral | system | web | bundle publication, stored draft/navigation state | +| dev and reload | Detect changed TUI code and replace its sandbox | Manifest version; one-second polling | ephemeral | system | TUI | bounded download, snapshot transfer | +| dev and reload | Accept local development commands without exposing a network-admin socket | Protected Unix socket; `0700` directory; POST body ≤16 KiB; 120-second timeout | ephemeral | owner, authorized local process | server/local host | filesystem permissions, command validation | +| dev and reload | Restart through the supervisor and reopen previously running sessions | Running-session IDs and supervisor result | until restart handoff | owner/system | server, session workers | supervisor outside core, worker startup | +| dev and reload | Retry transient operations interrupted by compatible code changes | Fresh execution context; at most 20 conflict retries | ephemeral | system | server, session worker | cancellation, reload/version detection | +| logs and notices | Capture diagnostics without altering actual stdout/stderr behavior | UTF-8 lines, original binary stream behavior; capture queue ≤256 KiB | until restart | system captures; owner reads diagnostic copies | server, session worker, detached hosts | stream tee | +| logs and notices | Redact secrets before storing diagnostic copies | Known/current/rotated literals plus structural credential patterns | until restart redaction set | system; owner sees redacted output | server, session worker, detached hosts | private configuration/env discovery | +| logs and notices | Bound retained diagnostic history | ≤128 sources; each ≤1,000 lines/256 KiB; lines >16 KiB omitted; partial lines >64 KiB discarded | until restart | owner | server | redaction, log capture | +| logs and notices | Preserve bounded detached-host logs across process exits | Atomic redacted JSON snapshot; private file; 50-ms publication debounce | persisted | owner; system writes | terminal host/dev supervisor, server | redacted log store | +| logs and notices | Surface operational failures separately from raw logs | One latest notice/source, ≤50 notices; source/time/message | until restart | owner | server, web, TUI | worker/dev/server failure reporting | +| logs and notices | Merge and dismiss duplicate owner notices | Notice identities; no log text in dismissal storage; success hides after five seconds | persisted error-dismissal IDs; ephemeral success notices | owner | web | log/dev notice feeds | +| logs and notices | Inspect logs without waking stopped workers | Source selector and retained snapshots | until restart or persisted detached snapshot | owner | server, web, TUI | diagnostic storage, not worker acquisition | +| logs and notices | Show short local TUI feedback | Latest three messages; repeated last message becomes a count | ephemeral; cleared on navigation | local TUI user | TUI | command/action outcomes | +| system operations | Expose readiness without requiring credentials | Tiny health response; only accepted hosts/origins | ephemeral | system, local/remote launcher | server | listener/access-origin validation | +| system operations | Run only one server for a data directory | Process lock and private sessions directory | ephemeral lock; persisted directory | system | server | local filesystem | +| system operations | Close sockets, workers, watches and storage on shutdown | Resource handles and shutdown ordering | ephemeral cleanup | system | server, session worker | all active resource lifetimes | + +| cross-cutting need | concrete behavior to preserve | affected functions | +|---|---|---| +| Identity provenance | Transport assigns identity; typed speaker markers cannot replace it; historical display names remain attached to entries | authentication, human input, shell entries, draft ownership | +| Permission distinctions | Guest read is not workspace/file-index access; guest write permits tool-driven filesystem actions; terminal access remains owner-only | conversations, files, workspace, execution, terminals | +| Revocation | Existing connections and retained live access end when a device/grant is revoked; reconnect does not silently restore old authority | sharing, device access, worker routing, web/TUI | +| Durability classification | Distinguish durable work/transcript, catalog/settings, local files, browser-tab state, browser-wide preferences and memory-only UI state | every domain | +| Stable operation identity | Keep submission IDs across uncertain acknowledgements; deduplicate recovered failure notices | sending, reconnect, UI replacement, model errors | +| Cancellation ownership | Navigation cancels target-bound local grants; shutdown leaves resumable durable work; removal may need actual work termination | launcher actions, reload, workers, long-running work | +| Safe location changes | Inspect under an admission fence; commit catalog/cwds; notify consumers; retire old environments; recover from timeout | relocation, clone, workspace/prompt watches | +| Cleanup beyond core | Core pauses/stops workers; session-store copying, blob copying, archive/delete cleanup and read-mark persistence remain supplied outside `src` | clone, archive, delete, unread tracking | +| Snapshot recovery | Reconnect reconstructs subscriptions; stale page replies are ignored; loaded old history is retained only under its stated cache lifetime | links, history, search reveal, workspace | +| Explicit bounds | Frame, backlog, call, bundle, snapshot, attachment, preview, log, preference and execution-output limits are separate budgets | transport, files, diagnostics, TUI | +| Secret handling | Redacted diagnostic copy is not a promise that raw process output is redacted; launcher secrets never enter downloaded UI code | logs, authentication, local actions | +| UI state survival | Preserve drafts, uncertain submissions, reading anchors and review drafts deliberately; browser attachments remain memory-only | navigation, reload, stop, reconnect | +| Safe rendering | Transcript HTML does not execute; safe-file preview excludes active formats; executable artifacts require their separate sandbox policy | Markdown, files, artifacts, terminal output | +| Configuration races | Watch before loading; retry partial saves; retain last valid state; serialize writes; in-flight work keeps its captured settings | models, prompt resources, environments | +| Read-only diagnostics | Looking at status/logs must not create worker activity or resume archived work | status, logs, indexing | +| Optional functionality | UI discovery and unavailable-state behavior must work when a capability is absent | completion, prompt expansion, logs, terminals, model controls | +| Accessibility and platform behavior | Keyboard focus, Escape/Back, mobile navigation, resizing, selection, terminal mode cleanup | web shell, TUI, viewers, workspace | +| Development atomicity | Failed reload does not publish candidate UI code or change future worker selection | dev controls, bundles, worker spawning | + +| function duplicated or overlapping across plugins today | current locations/evidence | distinction relevant to the map | +|---|---|---| +| Compose and deliver human conversation text | Chat, chat-mode and conversation-related UIs; shared core author/draft helpers | Different conversational modes repeat composition behavior; preserve attribution, attachments, request identity and restoration, not separate implementations | +| Present provider-login progress and prompts | Login web and TUI presentations; login flow remains server-owned | Same user function across two frontends; not two authentication authorities | +| Browse/select a working location | Environment management, session creation and session options; shared directory picker and TUI location helpers | Repeated entry points already reuse substantial code; do not mistake them for separate filesystem services | +| Observe configuration/resource files | Models, environments and prompt loading use the shared core observer | Shared watch machinery already exists; each still owns validation, last-valid state and domain-specific errors | +| Browse/search workspace filenames | Workspace browsing and file-completion indexing | Same tree and confinement requirements, different result shapes and bounds; core completion consumes the index rather than scanning independently | +| Show files and generated artifacts | Files/artifacts/workspace UIs plus transcript tool cards | Overlapping preview/download/navigation functions; active artifact previews and ordinary files intentionally have different execution policy | +| Show failures and operational outcomes | Login, environments, sessions, chat, logs and dev controls; global owner lane merges log/dev duplicates | Errors need domain context, but duplicate delivery of the same event should not produce multiple notices | +| Present conversation work and usage | Chat, status, session lists, child-agent views and shared core statistics/renderers | Multiple projections of the same underlying activity; distinguish conversation activity, worker liveness and session-wide totals | +| Apply session permissions at different transport boundaries | Files/artifacts/workspace HTTP access, worker operations and client affordances | Repeated checks are intentional enforcement layers, not redundant authorization that can simply be removed | +| Render tool and transcript content | Chat web/TUI, script/nested-call displays, file/shell renderers | Shared transcript data coexists with frontend-specific renderers; preserve equivalent meaning without requiring identical rendering | diff --git a/apps/p2/docs/p3-functions/environments-workspace.md b/apps/p2/docs/p3-functions/environments-workspace.md new file mode 100644 index 000000000..4574dda7e --- /dev/null +++ b/apps/p2/docs/p3-functions/environments-workspace.md @@ -0,0 +1,185 @@ +# Functionality map: environments, execution, terminals, workspace, file index, ports, radius remote access + +Read-only map of what the app must do in environments, execution, terminals, workspace, file index, ports and remote access, drawn from the p2 code. + +**Legend.** Who: O = owner; GR = guest read; GW = guest with write access; W = session worker; S = system. Where: Srv = server; Wkr = session worker; Web; TUI; Rem = remote env; Host = detached terminal host. Durability: E = ephemeral; R = until restart; P = persisted; T = transcript-bound. + +### Environments (local, SSH) + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| env | list configured remote envs with config, test result, session count | ≤ tens × ~300 B | P (config), R (test) | O | Srv, Web, TUI | session catalog | +| env | add or edit env: name, host, user, port, identity file, shell path, login shell, shell env vars | ~300 B each | P | O | Srv, Web | validation | +| env | validate: name format; no leading `-`, whitespace or control chars in host/user; no `"` in identity file | – | – | S | Srv | – | +| env | invalid config file keeps last valid set and shows an error | error string | R | O | Srv, Web, TUI | config watch | +| env | config file edited by hand is picked up (file watch, polling fallback) | – | R | S | Srv | file watch | +| env | delete env, refused while any session (archived too) uses it; removes its managed key | – | P | O | Srv, Web | session catalog | +| env | env revision = hash of config; part of env identity ("same id sees same files") | 12 hex chars | derived | S | Srv, Wkr | – | +| env | generate an ed25519 key per env, kept in the private data dir, adopted as identity file; existing key kept | key files | P | O | Srv, Web | key tool | +| env | upload a private key (≤ 16 KiB, passphrase keys refused), derive the public key | key files | P | O | Srv, Web | key tool | +| env | show the public key to install on the host (lookup debounced 300 ms) | ~100 B | – | O | Web | – | +| env | connection test: connect like a session (detect, deploy daemon, start), latency, platform, arch, home, version, watch mode, warnings | ~500 B | R (per revision) | O | Srv, Web, TUI | SSH, daemon deploy | +| env | unknown or changed host key: show fingerprints, trust after confirm, or forget | key lines | P (own known_hosts) | O | Srv, Web, TUI | test | +| env | host keys kept apart from the user's known_hosts, aliased per env name | file | P | S | Srv, Host | – | +| env | warm SSH connections of all envs when the folder picker opens (drop?: opens every env, even unused ones) | – | E | O | Srv | browse connection | +| env | browse an env's directories: `~` and relative paths, symlinks followed, ≤ 10k entries scanned, ≤ 1000 shown, "truncated" flag | ≤ ~100 KB | E | O | Srv, Web | browse connection | +| env | browse this device's directories (same caps) | same | E | O | Srv, Web | – | +| env | one shared browse connection per env, closed after 10 min idle or on config/key change | SSH connection | R | S | Srv | – | +| env | recent directories per env offered in the picker | from session list | derived | O | Web | session catalog | +| env | resolve a working directory for a new or moved session, optionally create it, check it is a directory | path | – | O | Srv | browse connection | +| env | TUI `/envs`: list, mark current, Enter moves the session, Ctrl+T test, Ctrl+K review host key, Ctrl+F forget | – | E | O | TUI | relocation, test | +| env | sessions on a changed env restart their worker once idle (no running tasks, same generation) | pending map ~ sessions × 40 B | R | S | Srv | worker status, catalog | +| env | separate copy of test results (drop?: duplicates the persisted test state) | – | R | S | Srv | – | +| env | worker opens an env-config watcher it never reads (drop?: unused) | watcher | R | S | Wkr | – | +| env | full list as a method next to the live list (drop?: same data as the live state) | – | – | O | Srv | – | + +### Execution + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| exec | local execution env at the session cwd | – | R | W | Wkr | – | +| exec | remote execution env over SSH (pi-env daemon), connects at first operation; offline is that operation's error | SSH connection | R | W | Wkr, Rem | env config, daemon deploy | +| exec | a remote session never falls back to local tools; unknown env refuses | – | – | S | Wkr | – | +| exec | daemon binary per remote platform/arch, deployed over SSH; local dev build or override variable | binaries ~ MB | P (remote) | S | Wkr, Host, Rem | – | +| exec | subagent with its own cwd on the same connection | – | R | W | Wkr | – | +| exec | path rules of the remote OS (Windows/POSIX), remote home `~` | – | – | S | Wkr | connection | +| exec | prompt names the SSH host the tools run on | string | – | S | Wkr | env config | +| exec | move a session between this device and envs, or between directories: history stays, files not copied; refused while work runs (fence) | location | P | O | Srv, Wkr, Web, TUI | dir resolve, idle check | +| exec | after a move, watchers reopen at the new location; old envs close after commit | – | R | S | Wkr | location change | +| exec | private data dir denied to file views on this device (canonical check) | path | – | S | Wkr | – | +| exec | active calls counted so a move or idle stop waits | counters | R | S | Wkr | – | + +### Terminals + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| term | list a session's terminals (title, cwd, env, size, created, exit status) | ~200 B each | R (host lifetime) | O | Host, Srv, Web | – | +| term | open a terminal at the session's cwd and env; refused for archived sessions | PTY | R (host) | O | Host, Srv, Web, Rem | session info, env config | +| term | terminals survive server and worker restarts (detached host, one per data dir, process lock) | – | until host exits | S | Host | – | +| term | host exits after 10 min with no terminals and no clients | – | – | S | Host | – | +| term | attach to an existing terminal; only one of this session | – | E | O | Srv, Web | list | +| term | attaching gets the current screen (headless emulator, 10k-line scrollback snapshot) | ≤ MB | R | O | Host | emulator wasm | +| term | several viewers share one terminal; resize follows; last viewer's theme answers color queries; light/dark change reported | – | R | O | Host, Web | – | +| term | output flow control: 256 KiB window, acks per 64 KiB, a viewer that falls behind gets a fresh snapshot; 8 MiB backlog cap disconnects | counters | E | S | Host, Srv, Web | – | +| term | input typed before the program starts is buffered (≤ 8 MiB) | bytes | E | O | Host | – | +| term | close: asks first if a foreground job runs (shows its name), force close kills | – | – | O | Host, Web | – | +| term | title tracked from the program | string | R | S | Host | – | +| term | one connection per env, reused, closed when no terminal runs on it; daemon checked for PTY support | connection | R | S | Host, Rem | env config | +| term | byte channel opened with a single-use token (30 s), owner only; JSON on the control channel, bytes only on the byte channel | token | E | O | Srv, Web | – | +| term | "host runs older code" notice; restart ends running terminals | version hash | E | O | Srv, Web | host version | +| term | host startup and failure log, bounded and redacted | ≤ KB | P (bounded) | O | Host, logs | logs | +| term | web: side or bottom placement, panel height, key-row toggle | localStorage | P (browser) | O | Web | – | +| term | web: on-screen key row with sticky Ctrl/Alt, clipboard paste, long-press native menu, mobile IME edits retyped, touch scroll with fling, mouse reporting | – | E | O | Web | – | +| term | web: parse output in time slices per frame so floods don't freeze the page | – | E | S | Web | – | +| term | TUI terminals: none today | – | – | – | – | – | + +### Workspace (files, git, review, uploads) + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| ws | list a folder: dirs first, gitignored entries marked, ≤ 2000 entries, `.git` hidden | ≤ ~200 KB | E | O, GW | Wkr, Web, TUI | exec env, git | +| ws | read a file: ≤ 1 MiB text, binary detected, truncated flag | ≤ 1 MiB | E | O, GW | Wkr, Web, TUI | exec env | +| ws | file as of a commit | ≤ 1 MiB | E | O, GW | Wkr, Web | git | +| ws | git status: branch, head, changes incl. untracked and conflicts, ≤ 5000 | ≤ 1 MiB | E | O, GW | Wkr, Web, TUI | git | +| ws | diff vs HEAD: context lines (3, 10, 25, full), untracked shown as added, unborn branch, ≤ 512 KiB | ≤ 512 KiB | E | O, GW | Wkr, Web, TUI | git | +| ws | commit log: paged, unpushed commits marked, upstream name | ≤ 512 KiB | E | O, GW | Wkr, Web | git | +| ws | one commit's message, changed files and per-file diff | ≤ 1 MiB | E | O, GW | Wkr, Web | git | +| ws | literal text search: smart case, include/exclude globs, ≤ 500 matches, ≤ 20 per file, works outside a repo | ≤ 256 KiB | E | O, GW | Wkr, Web, TUI | git | +| ws | all file paths for name filtering (git tracked+untracked, else walk ≤ 50k paths / 5000 dirs) | ≤ 4 MiB | E | O, GW | Wkr, Web | git | +| ws | change notifications (watch tree, HEAD, index, config; debounced 100 ms) → viewers reload | counter | R | S | Wkr, Web, TUI | file watch | +| ws | confinement: path rules, canonical root, no `.git`, private dir denied (also excluded from search) | – | – | S | Wkr | – | +| ws | git hardening: no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s timeout, output caps | – | – | S | Wkr | – | +| ws | view of the viewed conversation's own cwd (subagent worktree) | – | – | S | Wkr | conversation cwd | +| ws | download a workspace file (≤ 25 MiB; media by byte range; sandboxed attachment headers) | stream | E | O, GW | Srv, Wkr | read chunks | +| ws | preview: images, audio, video, rendered Markdown | – | E | O, GW | Web | download | +| ws | upload into a project folder: ≤ 25 MiB, 1 MiB chunks, temp file then rename, never overwrite, abort cleans up, writes serialized | ≤ 25 MiB | P (project files) | O, GW | Srv, Wkr, Web | exec env | +| ws | line or range review comments on a file or diff, quoting the line, sent as the user's next message (steer or follow-up) | ≤ KB each | P (browser) until sent | O, GW | Web | chat send | +| ws | review draft and open comment editor survive page reloads, per conversation | localStorage | P (browser) | O, GW | Web | – | +| ws | outdated comment marked when its line changed | – | E | S | Web | – | +| ws | UI: tabs (files, changes, history, search, terminal), wrap toggle, panel width, scroll memory, prefetch, split view | localStorage | P (browser) | O, GW | Web | – | +| ws | TUI `/files`, `/changes`, `/grep` (read-only) | – | E | O, GW | TUI | queries above | +| ws | TUI footer shows the git branch, refreshed on changes (500 ms debounce) | string | E | O, GW | TUI | status, changes | +| ws | an open panel counts as a worker viewer (keep-alive) | counter | R | S | Wkr | – | + +### Session files and blobs + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| blob | upload attachments to a session (≤ 25 MiB; image MIME types detected; stored outside the project) | ≤ 25 MiB each | P | O, GW | Srv, Web, TUI | – | +| blob | resolve attachment ids to files of that session only; ≤ 20 per message | metadata | – | W | Wkr | uploads | +| blob | copy attachments to the remote env before input is accepted; local copy kept for inline images | ≤ 25 MiB | P (remote) | S | Wkr, Rem | exec env | +| blob | download snapshots: immutable private copy (≤ 25 MiB) made by the agent's send-file tool; link shown to viewers | ≤ 25 MiB | P | W makes; O, GR download | Wkr, Srv | – | +| blob | serve a snapshot (cross-site requests refused; media by byte range) | stream | E | O, GR | Srv | – | +| blob | clone/fork copies the session's uploads, downloads and artifacts | dirs | P | S | Srv | – | +| blob | cleanup on archive or delete: none (drop? no: p3 needs it on session delete) | – | – | – | – | – | +| blob | server-side snapshot save method that nothing calls (drop?: unused) | – | – | – | – | – | + +### File index + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| idx | `@` completion: rank like pi (exact, prefix, substring, then fuzzy), dirs +10, limit 20 (max 100), kind filter | index ≤ 50k items | R (cache, kept 60 s) | O, GW | Wkr, Web, TUI | listing | +| idx | Tab directory listing for path completion | ≤ 2000 entries | E | O, GW | Wkr, TUI, Web | folder list | +| idx | completion refuses paths outside the cwd (`/`, `~`, `..`) | – | – | S | Wkr | – | +| idx | build from the env's own listing, else from git/walk; at most one rebuild per second while files change; discarded after a move | – | R | S | Wkr | exec env, change notifications | + +### Ports and forwards + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| port | list forwards with name, private/public URL, opener session title | ≤ 50 × 200 B | P | O | Srv, Web, TUI | radius origin | +| port | add (≤ 50; p2's own port refused; random id; removed then re-added = new URL), remove, toggle public (confirmed) | – | P | O | Srv, Web, TUI | – | +| port | agent tool `ports` list/open/close: top-level conversation only; owner-initiated or peer input only; idempotent | – | P | W (session from the worker's identity) | Wkr, Srv | input origin | +| port | sessions on remote envs cannot forward | – | – | S | Srv | session info | +| port | forwards a session opened close when it is archived, and at start for already-archived sessions | – | P | S | Srv | archive event | +| port | tool cards web and TUI (only Radius-shaped URLs shown) | – | T | O, GR | Web, TUI | – | + +### Remote access (radius) and proxy trust + +| domain | function | data | durability | who | where | depends on | +|---|---|---|---|---|---|---| +| radius | sign in to Radius by device code from a CLI; credential, server id, public origin in a private file | ~2 KB | P | O | CLI, Srv | – | +| radius | outbound relay connection: owner check, token refresh, heartbeat (20 s ping, 60 s timeout), backoff 1–30 s, signed-out recheck every 30 s, disable switch | socket | R | S | Srv | credential | +| radius | relayed browser requests become loopback requests marked with a per-process secret: owner, but treated as remote (no localhost-only powers); secret redacted from logs | secret | R | S | Srv | – | +| radius | flow control: HTTP credit, congestion pause, one socket per WebSocket channel; old relay protocol versions kept (drop?: if the relay is current) | – | E | S | Srv | – | +| radius | forward channels → `localhost:<port>`: Host/Origin/Location rewritten, 502 when nothing listens, unknown forward 404, anonymous only to public forwards, stream channels refused | – | E | S | Srv | forwards | +| radius | remember the brain's public origin for forward links, saved across restarts | string | P | S | Srv | – | +| radius | status lines in logs | – | E | O | Srv | logs | +| access | any value of the proxy header removes device (localhost) trust | – | – | S | Srv | – | +| access | proxy marker object that serves nothing (drop?: the rule lives in host code) | – | – | – | – | – | + +### (1) Cross-cutting needs + +| need | where it shows up | +|---|---| +| blob storage per session with copy on clone and cleanup on delete | uploads, download snapshots, artifacts; workspace downloads are streamed | +| chunked byte transfer between server, worker and browser, with ranges | workspace download and upload, attachment copy to remote | +| per-session execution env that follows moves and reopens watchers | workspace, file index, attachments, prompt, terminals (by session cwd and env) | +| resolved env config needed on server, worker and terminal host | SSH options, host-key file, shell env | +| long-lived processes outside the server with their own revision | terminal host, deployed remote daemons | +| connection pools with idle close and invalidation on config change | browse connections, terminal-host connections, worker env | +| keep-alive and activity: panel viewers, active calls, move fence, idle stop | workspace, file index, relocation | +| readers that must not start or keep a worker | session lists, env session counts, terminal list (served by the host) | +| cancellation tied to an HTTP disconnect, plus cleanup that is not cancelled | workspace upload abort | +| caps everywhere (sizes, counts, timeouts) | 25 MiB, 1 MiB, 2000, 5000, 50k, 10 s, 8 MiB backlog | +| bounded, redacted logs for separate processes | terminal host, radius secret | +| browser-local drafts | review comments, editor, placement and width | +| owner-only versus full-guest gating per function | terminals and envs owner only; workspace and index for full guests | +| secrets kept in private files, never in state | keys, radius credential | + +### (2) Functions duplicated across plugins today + +| function | copies | +|---|---| +| env config parsing and SSH options | envs server, envs worker, terminal host (copied modules) | +| SSH connection to an env | worker env, browse connection, test connection, terminal host | +| private data-dir denial and confinement | workspace backend, upload writer, file index | +| full file listing | workspace (git/walk), file index (env listing, then workspace) | +| download snapshot read/write | files server and send-file worker (both write the same directory) | +| upload handling | attachment uploads and workspace uploads (two routes, two caps, two layouts) | +| byte-range and media headers | files download route and workspace download route | +| session data-dir layout | files plugin and session clone/copy code | +| env test results | persisted test state and an in-memory copy | +| folder picker caps and symlink following | local listing and remote listing | +| session count per env | env list and delete check | +| remote OS path rules | envs, files worker, prompt | diff --git a/apps/p2/docs/p3-functions/models-prompt-shells.md b/apps/p2/docs/p3-functions/models-prompt-shells.md new file mode 100644 index 000000000..4177129ec --- /dev/null +++ b/apps/p2/docs/p3-functions/models-prompt-shells.md @@ -0,0 +1,169 @@ +# Functionality map: models, credentials, settings, prompt resources, code mode, coding tools, one-shot agents, logs, dev and reload, PowerShell, web and TUI shells + +**Legend.** +- **Dur** (durability): E = ephemeral, R = until restart, P = persisted, T = transcript-bound. +- **Who:** O = owner, GR = guest read, GW = guest write (send access), W = worker, S = system. +- **Where:** Sv = server, Wk = session worker, Web, TUI, L = TUI launcher (visitor's machine), RE = remote environment. + +## Models, credentials, settings + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| List the models of signed-in providers (the faux provider in demo mode) | ~hundreds of entries: id, name, thinking levels, context window; ~100 KB | R | O | Sv, Web, TUI | credentials, models.json, catalog cache | +| Compose custom providers, models and overrides from models.json (baseUrl, api, headers, compat, cost, thinking map, input limits) | models.json, KB | P (owner-edited file) | O edits, S reads | Sv, Wk | config values | +| Resolve config values: `!cmd` (10 s timeout), `$VAR`, `${VAR}`, `$$`; register the result as a secret | — | E | S | Sv, Wk | redaction | +| Hot-reload settings.json, models.json, models-store.json and auth.json: watch, re-read once on a half-written file, keep the last valid state, report errors per file | fingerprints | R | S | Sv and every Wk separately | file watch | +| Fall back to a 2 s poll when the watcher fails | — | R | S | Sv, Wk | — | +| Default model for new sessions: setting, then env `P2_PROVIDER`/`P2_MODEL`, then the first signed-in model; thinking level clamped to the model | settings | P | O sets, all read | Sv, Web, TUI | catalog | +| Set the default model and thinking level ("set as default" from a picker) | settings.json | P | O | Web, TUI → Sv | settings | +| Model scope for cycling: `enabledModels` patterns, globs, `:level` suffix, model ids that contain colons; set and clear | ≤500 patterns | P | O | TUI only (none on web) | catalog | +| Pick the conversation's model and thinking level | conversation agent config | T | O; guests see a read-only line | Web, TUI → Wk | conversations | +| Cycle models (Ctrl+P / Shift+Ctrl+P over the scope) and thinking levels (Shift+Tab) | — | E | O | TUI | scope | +| Refuse an unknown model at session create, at configure, and at worker start (visible error with Retry) | — | E | S | Sv, Wk | catalog | +| Model access for runs: a request in flight keeps the configuration generation it started with | provider objects | R | W | Wk | credentials | +| Run settings read at each decision: stream (transport, timeout, retries, cache retention), retry, compaction, tool execution, steering and follow-up modes | small | P | O edits file | Wk | settings | +| Catalog cache (models-store.json): restore offline, refresh from the network on demand | per provider, KB–MB | P | S | Sv writes, Wk reads | — | +| Providers with their login methods and signed-in type; never secrets | small | R | O | Sv, Web, TUI | catalog | +| Sign in with an API key (secret prompt) or OAuth (URL, device code, pasted code, account select); 10 min deadline; one login at a time server-wide; it belongs to the starting device | prompts, notices | E (credential P) | O | Sv, Web, TUI | credentials, questions | +| Remote owner: device-code login selected automatically | — | E | S | Sv | login | +| Answer, cancel or dismiss a login prompt; select answers validated; secret answers redacted before use | — | E | O (starting device) | Web, TUI | redaction | +| Abandon a login when its owner closes the sheet everywhere or disconnects | — | E | S | Sv | presence | +| Device identity for providers that need one (device.json) | UUID | P | S | Sv | — | +| Sign out of a provider (credential and cached catalog removed); refused while a login runs | — | P | O | Sv | credentials | +| Refresh a provider's models; report the count | cache | P | O | Sv | catalog | +| Credentials in auth.json: locked, validated, fail-closed at startup, last-good kept on a bad edit, keys from env or command, never sent to clients | KB | P | S; written by server and workers (OAuth refresh) | Sv, Wk | locked file | +| Owner display name on new owner entries (≤40 chars, no `<>` or control characters) | — | P | O | Wk | settings | +| Code mode on/off, applied live in every worker | bool | P | O (file edit) | Wk | settings | +| Show a model error in the transcript as a display-only entry, deduped and recovered at start | small per error | T | O, GR | Wk | conversations | +| Per-model cost rates and tiers for the cost panel | — | R | O, GR | Web | catalog | +| Show settings, model and credential load errors | strings | R | O | Web, TUI | — | +| (drop?) Fake provider with scripted prompts inside product code | — | E | S | Sv, Wk | move to test fixtures | + +## Prompt resources + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Global AGENTS.md (data directory, always local) plus AGENTS.md from the filesystem root down to cwd, read in the session's environment | ≤ tens of KB | R | S | Wk, RE | environment files | +| Discover skills: `<data>/skills` or `~/.pi/p2/skills` on a remote, plus `<cwd>/.pi/skills`. Rules: a SKILL.md directory is one skill; root `.md` files count; frontmatter name and description; skip hidden and node_modules; first of a name wins | list of skills | R | S | Wk, RE | environment | +| Skills block in the system prompt; `disable-model-invocation` skills hidden from the model | KB | R | S | Wk | discovery | +| Prompt templates: global and `<cwd>/.pi/prompts` `.md` files, `description`, `argument-hint` | list | R | S | Wk, RE | environment | +| Watch all of these (recursive, directory creation); poll fallback 30 s remote, 2 s local; 60 s load timeout; keep the last valid state plus an error | — | R | S | Wk, RE | file watch | +| Reload from the new cwd and machine after a relocation | — | R | S | Wk | location | +| A local session's first message waits for the first load | — | E | S | Wk | input gate | +| App guide section (app source, docs, config, skills paths) | text | R | S | Wk | — | +| List templates and skills for `/` completion | names, descriptions | R | O, GW (not viewers) | TUI; web has no completion UI | — | +| Expand `/name args` (bash-style args, `$1`, `$@`, `${@:N:L}`, defaults) and `/skill:name args` (SKILL.md read at invocation) on the worker before send | text ≤1 MB | E | O, GW | Wk; Web composer, TUI | environment | + +## Code mode, coding tools, one-shot agents, PowerShell + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Run model-written JS (QuickJS VM, worker thread) that calls the conversation's tools as nested durable calls | script, journal | T | model in turns by O or GW | Wk | conversations | +| Resume an interrupted script by replaying its journal; same VM fingerprint (fixed 256 MiB heap), else it ends with "sandbox changed" | journal | P | S | Wk | Durable | +| Script deadline; output bounded to the token budget, overflow spilled to a file | — | T | S | Wk | — | +| Offer or withdraw the tool live via the setting; a running script keeps its tool | — | P | O | Wk | settings | +| (drop?) "Removed mid-script" report, then abort; only a reload artefact | — | E | S | Wk | — | +| Code mode tool card: script, nested calls, agent rows | — | T | O, GR | Web, TUI | renderers | +| read, write, edit, bash in the session's environment; send access permits tool actions (not filesystem isolation) | output caps | T | O, GW | Wk, RE | environments | +| Preamble (speaker rules) and cwd section ("tools run on remote env X, SSH host Y") | text | R | S | Wk | location, environments | +| Tool cards: bash, read, write, edit (diff), grep, find, ls | — | T | O, GR | Web, TUI | — | +| One-shot agent (scripts only): fresh child conversation, prompt plus JSON Schema; parse plain or fenced JSON, validate, up to 3 retries with feedback; resolves to the value | child transcript | T (owned by the call) | W (script) | Wk | conversations, models, code mode | +| Agent model, thinking level and tool subset (default: the caller's); validated before any commit | — | T | W | Wk | catalog | +| Reattach after a restart (task document holds the child id; submissions `agent:N` deduped) | id | P | S | Wk | Durable | +| Aborting the call aborts the child's work | — | E | S | Wk | ownership | +| Child read-only for people (Stop allowed) and follows the session cwd | policy | T | S | Wk | conversations | +| Agent rows on the script card (label, model); transcript opens read-only; not in the sidebar; cost counts toward the session | metadata | T | O, GR | Web, TUI | session cost | +| PowerShell tool only on Windows: local from the platform, remote from the environment's OS report; ask again every 30 s if unreachable and after a move | bool | R | S | Wk, RE | environments | + +## Logs and notices + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Copy (tee) server stdout and stderr into a redacted ring without changing real output | 1000 lines / 256 KiB per source, 128 sources, lines ≤16 KiB | R | S | Sv | redaction | +| Worker stdout, stderr and lifecycle under `session:<id>`; stderr tail (4000 chars) for the crash message | same bounds | R (server memory) | S | Sv, Wk | — | +| Terminal host and dev supervisor publish redacted snapshot files; the server imports changes every 300 ms with dedupe; a final synchronous snapshot on exit | ≤2 MiB files | P | S | Sv, detached processes | file format | +| (drop?) Importing those files by polling; they could be pushed over IPC | — | — | — | — | — | +| Redact secrets: config files (auth, radius, models, settings, envs, owner key/token), secret env vars, resolved config values, login answers, codes and URLs, auth headers, Bearer/Basic, URL passwords, `sk-` keys; old secrets kept after rotation | secret set | R | S | Sv, Wk, detached | — | +| View a source: filter, follow the tail, copy the visible lines, switch source; never starts a worker | ≤1000 lines | R | O | Web, TUI | — | +| Owner notices: latest error per source (≤50) from host errors, worker failures, refused reloads and stderr `Error:` lines; dismiss | — | R | O | Sv, Web lane, TUI status lines and `/notices` | — | +| Notice summary when a TUI opens; new notices shown as they arrive | — | E | O | TUI | — | +| Dev notices merged with error cards; dismissal remembered per browser (identities only) | — | P (browser) | O | Web | — | +| Worker start failure: error, Retry, Show logs; the failure is cached so background references do not respawn the worker | — | R | O | Sv, Web, TUI | sessions | + +## Dev and reload + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Watch plugin and runtime files; mark dirty only; track the files each bundle and host actually loaded | dirty set | R | S | Sv (dev) | — | +| Reload the dirty or named plugins on the server and running workers; rebuild web and TUI bundles; refuse a removal while it is held; report the selection or the reason | — | R | O | Sv, Wk | bundles | +| Supervised restart (graceful, SIGKILL after 10 s); restore non-archived workers; no duplicate inputs | running set (supervisor memory) | R | O | supervisor | — | +| (drop?) Seeding running sessions through `P2_DEV_RESUME` | — | — | — | — | — | +| Dev status and buttons (GET/POST `/__p2_dev`); "Server code changed" list | status | R | O | Web, TUI (consent) | notices | +| CLI `npm run reload [-- names] [--restart]` over a 0600 Unix socket | — | E | account | host | — | +| `reload_plugins` agent tool: dev server only, local session on the p2 checkout, root conversation, owner-initiated turn (attribution from the calling entry, nested calls walk up to it), not replayed after a crash; plus a prompt section about it | — | E | O-initiated | Wk | turn attribution | +| Web: poll the manifest every 5 s while visible; reload on a new version or a newer link protocol | — | E | O, GR | Web | — | +| TUI: poll the hash every 1 s; prepare a new worker; carry a ≤1 MiB snapshot (viewed identity, drafts, pending send ids, reading positions, toggles); swap; `/reload` forces | — | E | O, GR | TUI, L | — | +| Serve bundles: content-named files, old files kept briefly, lazy first build, a failed build keeps the previous one; owners and admitted guests only | ≤8 MiB | R | O, guests | Sv | — | + +## Web shell + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Layout: sidebar (grouped by where tools run, pinned first, most recently active directory first, working and unread markers), main area, phone drawer, collapse state | — | browser | O; guests see their session | Web | sessions | +| Navigation stack: `/s/<id>/c/<n>` URLs, modal layers with locators, workspace and terminal carry-over, scroll memory, sessionStorage, Android CloseWatcher, Escape | — | tab | all | Web | — | +| Composer state kept across navigation: draft (also in sessionStorage), attachments, reply target, Stop notice | — | tab | O, GW | Web | — | +| New-session choices kept while browsing | — | E | O | Web | — | +| Theme per browser (system/light/dark), no flash on load, synced across tabs | — | P (browser) | all | Web | — | +| Reconnect with backoff ≤5 s; detect an ended share (HEAD /) and stop reconnecting | — | E | all | Web | access | +| Installable app (manifest, icons), CSP nonce | — | — | all | Web | — | +| Listing cache for Back and revisits | — | E | all | Web | — | + +## TUI shell + +| function | data | Dur | Who | Where | depends on | +|---|---|---|---|---|---| +| Sign in: loopback is the device owner; remote HTTPS owner link or share link (display name) redeemed into a per-origin cookie (0600); an ended share exits and is forgotten | config.json | P (visitor) | O, guest | L | access | +| Fetch a bounded, SHA-256-checked bundle; run it in a permission-dropped worker (no fs, net or env); watchdog at 512 MiB RSS | ≤8 MiB | E | O, admitted guests | L, TUI | bundles | +| Filter output (safe sequences, 64 KiB frames); track terminal modes and restore them on exit, crash and swap | — | E | S | L | — | +| Actions under a grant (own key or consent screen): upload a file or clipboard image, paste, copy, open URL, download, external editor, suspend, save prefs (diff), dev reload, sign out (this or all owner devices) | — | E | O (guests some) | L | — | +| Visitor preferences: pi keybindings, settings and themes (read only), p2 prefs.json, watched; server defaults (tui.json) below them; privileged bindings never from the server | KB | P (visitor, server) | visitor, O | L, Sv | — | +| Query terminal colors (OSC 10/11/4, mode 2031) | — | E | S | L | theme | +| Commands: pi ids, contexts, rebinding, conflict check, F1 palette, `/hotkeys`, `/settings`, `/keybindings`, `/quit`, `/reload`, thinking and tool-output toggles | — | E | all | TUI | — | +| Fullscreen layout; reading position kept while output grows; drafts per (session, conversation) | — | E | all | TUI | — | +| Reconnect with backoff; references rebind | — | E | all | L, TUI | — | + +## (1) Cross-cutting needs + +| need | where it shows up | +|---|---| +| Config files with a single writer | settings, models and auth files are watched by the server and every worker; auth.json is written by both | +| Secrets | central redaction; credentials never reach clients; workers need auth for model calls | +| Watching files in an environment (local or SSH) | prompt files, OS probe, workspace, file index; needs poll fallback, timeouts, last-valid state, re-arm after relocation | +| Background work with no viewer | scripts, one-shot agents, logins (10 min), reloads; they keep the worker alive and are aborted with their owner | +| Cancellation chains | script → nested calls → child conversation; login cancel or abandon; reload aborts | +| Readers that must not start workers | logs for `session:<id>`, sidebar markers, cached failure state | +| Bounded buffers | logs, TUI snapshot (1 MiB), frames (4 MiB / 64 KiB), tool output, bundles (8 MiB) | +| Turn attribution | owner, guest or peer, for privileged tools (`reload_plugins`) and the speaker preamble | +| Client code delivery | version push, swap with state carry-over, guest-limited assets | +| Client-side consent | the TUI grant model | +| Deterministic fake provider | for tests | + +## (2) Functions duplicated today + +| duplicated function | places | +|---|---| +| Watch, last-valid state, retry once | models config loader, prompt watcher, TUI preferences watcher, dev watcher, logs file import | +| Config and credentials loaded per process | server plus each worker; both write auth.json | +| Drafts | web sessionStorage, TUI snapshot, server drafts | +| Model lookup and choice resolution | chat, subagents, agent, sessions; thinking clamp in config.ts and subagents | +| Turn attribution | conversations' origin.ts, copied into dev-tools | +| Following the session location | prompt, PowerShell, dev-tools, workspace, file index | +| "Offer tool only when" switches | code mode, PowerShell, dev-tools | +| Registering secrets for redaction | models, login, radius, server env scan, worker | +| Notice channels | log notices, dev notices, model-error entries, TUI grant notices, web notice lane | +| Staleness checks | web poll every 5 s, TUI poll every 1 s, protocol-mismatch reload | +| Reconnect backoff | web, TUI, `useObject`'s 500 ms re-hold | +| Log source labels | web and TUI | +| `/` expansion | web composer and TUI | +| Themes | web and TUI, separately | +| Environment file reading | prompt, workspace, file index | diff --git a/apps/p2/docs/p3-functions/sessions-access.md b/apps/p2/docs/p3-functions/sessions-access.md new file mode 100644 index 000000000..9595cf35d --- /dev/null +++ b/apps/p2/docs/p3-functions/sessions-access.md @@ -0,0 +1,160 @@ +# Functionality map: sessions, sharing and guests, devices and owner access, status and presence, search, subagents, provider logins + +**Legend.** +- Durability: E ephemeral · R until restart · P persisted · T transcript-bound (forks and rewinds with history). +- Who: O owner · Gr guest read · Gw guest write · W worker · S system. +- Where: Srv server · Wk session worker · Web · TUI · Env remote environment · CLI. + +## Sessions + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| create session in a directory, local or on an environment; optional create-dir; optional title; model and thinking default to the last used | row ~300 B | P | O | Srv, Web, TUI, Env | dir check, model list | +| check directory: absolute, `~` expanded, is a directory; remote: resolved on the environment | path | E | O | Srv, Env | environments | +| refuse a model the configuration doesn't know | model ref | E | O | Srv | model catalog | +| list all sessions live: pinned first, then recent; never starts workers | N × 300 B | P | O; Gr: sessions shared with them, not archived | Srv→Web, TUI | — | +| suggest default cwd and environment for a new session (most recent active) | derived | E | O | Srv | list | +| rename: 1–120 chars, whitespace collapsed; an explicit title is never auto-replaced | title, edited flag | P | O | Srv, Web, TUI | — | +| automatic title: first user message, line breaks to spaces, 100 chars | text | P | S | Srv | message indexing | +| agent renames the session via a tool (root conversation only; tool explains when to rename) | title | P | W | Wk→Srv | rename | +| updated-at from message activity, never moves back; a move counts as activity | timestamp | P | S | Srv | message feed | +| pin / unpin; pinned filter in TUI | bool | P | O | Web, TUI | — | +| model and thinking level follow the root conversation's configuration | model ref | P | W | Wk→Srv | conversations | +| archive: stop the worker whatever it does; end guest access; close session-owned port forwards; keep storage | flag, time | P | O | Srv | worker stop, revocation, ports | +| restore; guests regain access | flag | P | O | Srv | sharing | +| browse and search archived sessions separately | — | E | O | Web, TUI | search | +| relocate (cwd and/or environment) | location | P | O | Srv, Wk, Env | worker lifecycle, subagents | +| relocate · guards: refuse while busy (name the running subagents); refuse a cross-machine move if a subagent has its own cwd | — | — | — | — | — | +| relocate · steps: fence the worker; update the record; move the running worker in place (60 s, else restart); release the fence on failure | — | — | — | — | — | +| clone: store plus artifacts/downloads/uploads under a new ID, worker paused, refused while work runs, title "(copy)"; dirs 0700, files 0600, no links, no overwrite; partial copy removed on failure | session files (MB) | P | O | Srv | worker pause, session files | +| fork before a user message of the main conversation: active context as of the entry before it, edits applied, head marker; title "(fork)"; failure archives the copy; subagent conversations not supported | entry drafts KB–MB | P | O | Srv + 2 Wk | history read, empty-seed check | +| delete session — p2 has none (needed: removes all per-session data everywhere) | all | — | O | all | every per-session store | +| worker start parameters from the session record; refuse archived (except an index rebuild without resuming); refuse an unconfigured environment | init | E | S | Srv | record, environments | +| start worker on demand; idle stop after 5 min without viewers, calls or tasks | — | R | S | Srv | keep-alive | +| failed worker start sticks with its error until the owner retries | error text | R | O retry; Gr sees no error | Srv, Web, TUI | — | +| worker generation fences stale reports | counter | R | S | Srv | — | +| TUI selector: current folder vs all, fuzzy title/cwd/ID, sorts threaded/recent/relevance, show IDs, confirm archive, `+ New session` | — | E | O (Gr list) | TUI | list, children, search | +| `/new [env:]path`, `/cd`, `/name`; completion of environment names and directories | — | E | O | TUI | directory listing | +| web drawer rows: busy, error, unread, subagents; Session sheet: rename, pin, share, cwd picker, archive (navigates away) | — | E | O | Web | status, children, read marks | +| catalog schema upgrades (drop? p3 starts empty) | — | — | S | Srv | — | + +## Access: owner, devices, guests, sharing + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| loopback without cookie = device owner (TUI, CLI, first page load) | — | — | S | Srv | — | +| loopback owner browser cookie: wins over a guest cookie; tells the owner's browser from a fresh profile on the join page | 32 B secret file | P | S | Srv, Web | — | +| browser requests from another site get no principal, except a top-level navigation | — | — | S | Srv | — | +| one-time sign-in token: new one per server start in a private file; `owner-link` prints the link; 10 min, single use, ≤32 outstanding, memory only | 32 B | R | local O mints; devices cannot | Srv, CLI, Web, TUI | — | +| sign-in page: token from the fragment or pasted; claim body ≤1 KB, same-origin | — | E | anyone with token | Srv | token | +| device cookie: signed (HMAC), per port, 30 days, checked per request (expiry, revocation, signature) | key 32 B | P | S | Srv | key file | +| sign out this device (revoked until its expiry) / all devices (rotate key, clear tokens); live connections end; TUI via launcher consent | revocation list | P | O | Srv, Web, TUI | revocation | +| one-time migration of the old cookie scheme (drop?) | — | P | S | Srv | — | +| share link per session and mode (read-only, read/write), ≤1 each, created on first copy; URL on the first tunnel host, else loopback | 43-char token | P | O | Srv, Web, TUI | session open | +| confirm a read/write link on a remote environment ("guests can run commands there") | — | E | O | Web, TUI | session env | +| reset link: new token, its guests lose access at once | — | P | O | Srv | revocation | +| join page: server-rendered, CSP nonce, title and access text; owner or already-joined guest redirected | — | E | anyone with link | Srv | session title | +| join: name unverified, cleaned, ≤40, unique per session ("Ann (2)"); ≤1000 guests per link; cookie 30 days HttpOnly Strict; body ≤2 KB, same-origin | guest ~200 B | P | link holder | Srv | link | +| same browser keeps its identity; a removed guest cannot mint a new one | — | P | S | Srv | — | +| guest grant per request: session, mode, revision; void while the session is archived or unknown | — | E | S | Srv | session state | +| revision bump on remove, reset, archive or restore ends live objects and sockets on server and workers | int | P | S | Srv, Wk | — | +| remove (block) a guest | flag | P | O | Web, TUI | revocation | +| a guest holds one grant per session; a browser joining more links gains their sessions; it lands in the session shared or joined last | grant row ~100 B | P | S | Srv | links | +| share with a person by email (while connected to Radius): they open the brain in Radius, in any organization, signed in with that email; no cookie, no link; Enter copies the session's Radius address | email on the guest row | P | O | Srv, TUI, Radius | Radius relay principal | +| report the emails with grants on unarchived sessions to Radius after every change and connect; Radius admits and lists the brain for them, and learns nothing else | email list | P (in Radius) | S | Srv→Radius | grants, archive | +| connected guests now (name, mode), per tab | — | E | O | Web, TUI | presence | +| guest reaches only its session: worker, files, subagents | — | — | S | Srv | grant | +| guest UI: name and access in the header; read-only replaces the editor; full mode can send, upload, stop | — | E | Gr / Gw | Web, TUI | grant | +| guest last-seen written, never read (drop?) | ts | P | S | Srv | — | +| copy a link from the TUI through the launcher's consent | — | E | O | TUI | — | + +## Status and presence + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| per session: running, error, live task count, busy conversations (foreground only), usage and cost | ~200 B per session | R | O all; Gr own, no error text | Srv→Web, TUI | worker reports | +| worker reports its task graph and usage on each change; retried after reload conflicts | — | E | W | Wk→Srv | — | +| worker keep-alive counts Durable tasks (today fed by the status reporter) | count | R | S | Wk | task graph | +| viewer count per session (unique per page or TUI) | — | E | O, Gr own | Web, TUI | presence | +| TUI footer: env:cwd (branch) • title › subagent; tokens, cache hit, context %/window, session total cost, model and thinking | derived | E | O, Gr | TUI | conversation view, catalog | +| `/session` details | — | E | O, Gr | TUI | footer | +| retry a failed worker; notify the failure once | — | E | O | Web, TUI | worker lifecycle | + +## Search + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| index user and assistant messages of root and subagent conversations; skip shell output, Durable internal inputs, sender headers | ~transcript text | P (derived, rebuilt on schema change) | W→S | Wk→Srv | message stream, conversation list | +| catch up from a per-conversation cursor after each commit; 200 per page; cursor moves after the whole range; idempotent | cursors | P | S | Wk, Srv | — | +| search: title/cwd substring plus literal message terms (all required, no regex or phrases); active or archived scope; ≤41 hits, snippet ≤300, query ≤500; TUI debounce 200 ms | — | E | O | Srv, Web, TUI | index, list | +| open a hit at its session and conversation | — | E | O | Web, TUI | — | +| rebuild one session's index: stop, drop, rescan; archived without resuming (drop? no UI seen in this area) | — | E | O | Srv, Wk | worker lifecycle | +| unread: latest root answer vs owner read mark, shared by all owner devices; marked when the root conversation is visible; seeded as read at first creation | 2 ints per session | P (marks), derived (answers) | O | Srv, Web | index | + +## Subagents + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| spawn named subagent: model, thinking (clamped), tool subset, optional own existing cwd, first message | registry ~200 B each | T | W (agent tool) | Wk, Env | models, conversations | +| send (steer / follow-up), configure, stop, archive/restore, status, models (≤200, 10 s) | — | T | W | Wk | — | +| fetch latest answer (scan ≤2000 entries) | — | E | W | Wk | — | +| report each answer to the main agent exactly once; withdraw a message stuck in the queue | tasks | T | S | Wk | background tasks | +| subagents cannot spawn subagents | rule | T | S | Wk | — | +| code-mode one-shot agents: read-only, listed on the script card | registry | T | W | Wk | code mode | +| children list per session for the sidebar without workers (name, model, thinking, tools, cwd, last entry, archived) | ~200 B per child | P (server copy) | O, Gr own | Srv, Web, TUI | worker reports | +| archive/restore from UI; refused for one-shots | flag | T | O, Gw | Web, TUI | — | +| `/agents`, `/main`, back-to-main link, TUI tool card | — | E | O, Gr | Web, TUI | — | +| follow session cwd on relocate unless own cwd | — | T | S | Wk | relocate | +| mirror of conversation model/tools into the registry (drop? duplicates conversation config) | — | T | S | Wk | — | + +## Models and logins + +| function | data | dur | who | where | depends on | +|---|---|---|---|---|---| +| providers with login methods and signed-in type | small | derived | any O | Srv, Web, TUI | model config | +| login by API key or OAuth: one at a time on the server | — | — | — | Srv, Web, TUI | credential store | +| login · prompts: text, secret, select, pasted code | — | — | — | — | — | +| login · notices: auth URL, device code, progress, info | — | — | — | — | — | +| login · limits: 10 min deadline; device-ID file | — | — | — | — | — | +| login · behaviour: device code auto-selected on remote connections; model list refreshed after | — | — | — | — | — | +| login · state: small, E (finished state kept until dismissed); visible only to the starting owner, others are refused | small | E | starting O | — | — | +| URL open and copy through the launcher | — | — | — | — | — | +| cancel when the starting owner leaves everywhere (sheet closed, disconnect); back navigation cancels | — | E | S | Srv | presence | +| cancel, dismiss | — | E | O | Web, TUI | — | +| redact codes, URLs and typed secrets from logs; credentials never sent to clients | — | — | S | Srv | logs | +| logout (refused during a login), delete cached models | auth file | P | O | Srv | — | +| refresh a provider's model list, return the count | model cache | P | O | Srv | — | +| `/login [provider]`, `/logout`, with completion | — | E | O | TUI | — | + +## (1) Cross-cutting needs + +| need | where it shows | +|---|---| +| readers that must not start workers | session list, status, children, unread, search all need server-side copies | +| worker→server reports with a generation fence | status, children, index feed | +| catch-up after commits (cursor plus wake signal) | search indexer, subagent reporter | +| per-session data across hosts with copy, archive, delete | store, artifacts, downloads, uploads, links, guests, index rows, read marks, children, forwards | +| presence and leaving | guests connected, viewer counts, login abandon, idle stop | +| keep-alive | tasks, calls, viewers | +| cancellation of multi-step operations that outlive the caller | relocate, clone, fork, index rebuild | +| revocation of live principals | guest remove/reset/archive, device logout | +| secrets | redaction, file mode 0600, no credentials to clients | +| server-rendered pages | sign-in, join (CSP nonce) | +| size caps and timeouts | title 120, name 40, 1000 guests, 2 KB/1 KB bodies, query 500, 41 hits, 10 min token/login, 60 s move, 5 min idle | +| blob directories per session | artifacts, downloads, uploads | +| launcher consent actions in the TUI | copy, open URL, sign out | + +## (2) Duplicated today + +| function | places | +|---|---| +| "archived?" checks | worker start, share open, relocate, clone, fork | +| title cleaning | session rename, agent tool | +| `~` and path resolution | create, relocate, TUI location parser, web picker | +| presence leases | guests, viewers, login abandon | +| subagent registry read | subagents, code-mode agent, search | +| generation-fenced report stores | status, children | +| commit catch-up loops | search, subagents | +| revocation fan-out | sharing archive, link reset, device logout | +| thinking-level validation and clamping | sessions, subagents | +| server-page rendering with nonce | join, sign-in | diff --git a/apps/p2/docs/p3-handoff.md b/apps/p2/docs/p3-handoff.md new file mode 100644 index 000000000..d2454fbe5 --- /dev/null +++ b/apps/p2/docs/p3-handoff.md @@ -0,0 +1,322 @@ +# Handoff: Chord v2 and p3 (read this first after every compaction) + +Maintained by the redesign session (`8b6ab5cc-8f95-421b-989a-de109636ff5a`, runs in p2 on the phone; its tools run on +slayer). Updated whenever the state changes, together with the decision log. Use `history_search`/`history_read` for anything +not written here. + +## 1. What we are doing + +Redesigning Chord (the object runtime) and p2 (Pi Mobile) from scratch as **Chord v2** (`packages/chord/src`) +and **p3** (`apps/p3`, later). Goal (owner): a minimal, powerful runtime with the fewest footguns, no memory leaks, the +best possible performance and easy-to-grok code (size is not a goal by itself: owner, 2026-10-27). + +## 2. The process (owner's) + +Chord v2 is built in **vertical slices** (`packages/chord/docs/slices.md`), each the thinnest end-to-end path +through the layers it needs. Per slice: a spec written by this session (`slice-N.md`), reviews of the spec (Astra, +Fable), the owner's approval, a build by a fresh Opus builder (never commits unless told), reviews of the uncommitted +code (this session, Astra, Fable), fixes, the owner's review, commit. Every slice adds runnable examples +(`test/examples/NN-topic.ts`); hot-path slices add benchmarks (`benchmarks.md`). Code follows the root +AGENTS.md section "Chord v2 Code Style". Decisions go into the log at once and into `design.md` in the same commit. + +## 3. Where we are (update this section) + +**UPDATE 2026-11-03 midday (supersedes what follows):** p2 runs on Chord v2: private `16329187c` (switch), phone +deployed 12:38 (backup `~/p2-backup-20261103-1235/`, data-report --compare only growth, verify-sessions clean). +Phone lesson: a pull without restart is not inert (the dev server hot-reloads changed plugins). Biome fix for the +phone's WASM fallback `6c425c2c5`. Upstream since: loud mistakes `5bd014b1c` (+ test race `0c7c74198`), workers type +fix `51bb8f5c0`, tutorial `24d3993b2` (Radius 01m6g8fnaqfdsa8ad21y2dfpy8; p2 chapter example waiting in +`/tmp/tutorial/p2-chapter-example/`), int table keys `d7704516e`. Footguns list `/home/badlogic/p2-port/chord-footguns.md` +(13 + 4 tutorial); owner: no trial app, p3 finds and fixes as we go. Open: C1 delete Chord v1 (`src/old`), C2 p2 TUI +on Chord's child-process sandbox, place the tutorial's p2 example, Chord gaps (React state across plugin reload vs +P4, one-time read forces restart, contract imports in tests, `invalid` not exported), i18n build, slice 9. + +**UPDATE 2026-11-03 night (history):** Chord v2 is the package root: upstream `e9c0ff4d8` (CI green), +private `ce43d89b8` (p2 imports v1 from `/old`); Chord docs now `packages/chord/docs/` (decisions.md, design.md, +slices.md), examples `test/examples/`. 59-chat on a table upstream `3c7a9ca0e`. Decision "p2 moves to Chord v2, then +v1 is deleted" (`d5c4b80eb`, owner yes to all 11 points; 7 = createHttp later). Running overnight (owner wants the +port done by morning, no phone deploy): `ui-back` (Opus; brief `/tmp/ui-back-brief.md`, report `/tmp/ui-back/report.md`; +worktree `pi-ui-back` + `/home/badlogic/bundled-ui-work`; reverts Cristina's `1cf20071f`, ports bundled-ui UI changes +back, bundled-ui builds from pi-private; UNCOMMITTED until the owner checks in the morning) and `p2-port` (Opus; +worktree `/home/badlogic/pi-p2-v2`, brief `/home/badlogic/p2-port/build-brief.md`, plan `/home/badlogic/p2-port/plan.md`; +P1-P3, S0 now; after ui-back, bring its result into pi-p2-v2 and tell p2-port to do S1-S8 with the UI; I make local +checkpoint commits there, never pushed until reviewed). Chord move scripts kept in `/home/badlogic/chord-root-move/`. +bundled-ui is public on purpose (jsDelivr needs it); its git history holds the UI source. + +**UPDATE 2026-11-03 (history):** slice 11 committed upstream `5bc414516` (CI green), merged into +private `46905bc1c`. Decided and recorded: i18n (`9c248717d`, built after slice 11: carry-over), Chord v2 becomes the +package's root (`1d81a6972`: src/old, /json and /watch subpaths, short Node subpaths, docs -> docs, no version +bump). schema-guard scripts deleted (owner). `chord-root` is doing the real run (brief `/tmp/chord-root-run-brief.md`, +scripts `/home/badlogic/chord-root-move/`): worktrees `pi-chord-up` (upstream part) and `pi-chord-priv` (p2's part); +then an Opus reviews package description/README wording; then commit upstream, merge into private, commit p2's part. +Next after that: i18n build, telemetry (slice 9), small examples; p3 work packages after the meeting. + +**UPDATE 2026-11-02 late night (history):** +- Slice 11 sandbox is a child process (decision "The terminal's sandbox is a child process", upstream `128912d24`, + design `10cf3264f`, slice-11.md `65c85bda2`). Opus rounds: recheck 2 (`/tmp/s11-recheck2-opus.md`: E7 getter on + process prototypes, E8 SIGUSR1, E9 attach before yes, E10-E13) fixed by audit-builder (round 3); recheck 3 running + in `s8-spec-opus` (`/tmp/s11-recheck3-opus.md`). pi-s11 is a local commit `wip slice 11` (made with --no-verify: + REDO the final commit with hooks, amend message), rebased on upstream/main (no conflicts), slices.md carry-over + cleaned and "What exists" gained 11-13, package-lock regenerated with npm 11.21.1 (`/tmp/npmlatest`, the repo's + lock needs a newer npm than node 24's 11.4.2) and left unstaged; `npm run check` clean; full Chord tests running + (`/tmp/s11-final-tests.log`). Then: commit (amend, PI_ALLOW_LOCKFILE_CHANGE=1 with package-lock.json), push + upstream, CI, merge into private. +- `chord-root` subagent (Opus) in `/home/badlogic/pi-chord-root` (worktree of origin/private): trial of making Chord v2 + the package root, old v1 to `src/old` or p2 (Durable and env import the root too), as a re-runnable script + (`/tmp/chord-root/`), brief `/tmp/chord-root-brief.md`, report `/tmp/chord-root/report.md`. Owner wants pi-private + merged (into pi) this week. + +**CURRENT STATE 2026-11-02 (night); the older paragraphs below are history.** +- **Owner's working mode now:** "go ahead, just do all the things. tired of reviewing": this session writes specs, + runs builds, reviews and fix rounds, commits and merges without asking per point; only genuinely new decisions go to + him (explain with code; one at a time). All subagents on `anthropic/claude-opus-5-5` (never Fable: "costs too + much"; `s8-spec-fable` is just a name, it runs Opus). Sol (`s8-spec-sol`) and fresh reviewers keep failing with + model_error on security reviews: drop them, use Opus (`s8-spec-opus`) alone if needed. +- **Committed upstream (CI green, merged into private `edf7b4bed`):** slices 10, 12 (`9823079ad`, plugin model), 13 + (`072e413bf`, app kit), the schema switch (pi-schema, TypeBox gone except coding agent; `6536f8012`, `be039fb31`, + `a6bde34`, `6291b9bcc`), example `60-session-sync` (`15f00a786`). p2 migrated to pi-schema in private; phone runs + `66641ca8c` (voice fixes round 2) + sts-server `08f6cf8d6` (partial transcripts) deployed on slayer. +- **Slice 11 (web and TUI) is building, uncommitted:** `audit-builder` in `/home/badlogic/pi-s11` (base upstream + `d6af4d18e`; later docs commits not imported: rebase at commit). Built: surfaces, `startPage`, React hooks, themes, + TUI launcher + sandbox, app-kit follow-ups (`startServer`, `exportData`/`importData` without stopping), example + `59-chat` (worker per channel, uploads), fix round 1 done. **Now:** the TUI sandbox becomes a **child process** + (owner decided after Opus escaped the hardened worker twice: `/tmp/s11-recheck-opus.md` E1 `Set.prototype.has`, E2 + `EventEmitter.prototype.emit` → real `process`): pipes not the terminal, env {}, `--permission`, + `--no-experimental-sqlite`, length-prefixed frames, allowlist kept as second layer, V8 eval-flag toggle dropped, + E3-E6 launcher fixes, escape tests ported, benchmark vs the worker (startup, 160 KB frame loop, key latency). + Then: Opus recheck/escape round, fix, commit (rebase on upstream, check, Chord tests, CI), merge into private. + Known gap: the child can still signal user processes (`process.kill`); in the Node gist. +- **Node gist for Matteo** (owner forwarded): https://gist.github.com/badlogic/a83cd80d0669dbcf77e029ac7b25ac96 + (`/tmp/node-gist/`): sqlite ignores fs permissions (and writes despite ERR_ACCESS_DENIED), fd/tty writable, real + `process` via EventEmitter.prototype; asks: sqlite checks, fd permission, per-Worker permissions, signals/env. +- **Decided today, recorded in `decisions.md` (upstream), not all folded into design.md yet:** plugin model points + 1-8 (folded §21), app kit points 1-5 (folded §23), slice 11 P3-P9, looks and themes, `chord.plugins` as manifest, + App kit naming and export, sync applies batches with the host stopped, p3 meeting summary (Armin), p3 notes + (subagent `context: "inherit"`, `/tree` as in-session forks). +- **Open with the owner:** i18n for the app kit (explainer `docs/i18n/`, Radius 01m6ek54jff7atbe4rrkg1w3zz v3: + `defineMessages` per plugin, shared `messages.ts`, `t.message` typed messages as data, `Translations` registry per + host, locale per user, servers send keys) - asked "record as decided?", no answer yet. Building it would follow slice 11. +- **Meeting doc for p3** (2026-11-03): `docs/p3-features/` (book.md with {{img}}/{{clip}} tokens; build with + `/tmp/p3-doc/inline.py` then the explainer build; pictures in `/tmp/p3-prep/shots/`), Radius 01m6efr04nezms4zx32p7pp0w1 + v2. Research files `/tmp/p3-prep/{features,extensions,envs,sync-example}.md`. Other explainers today: theming + (`docs/theming/`, 01m6ecp0ndf8hb1m0ea6dqy8jn), plugin primer (01m69waqspfwwt21bjhvsp5w28), schema charts + (01m69x3dz2fsg9k1r33vde7hv6 v6). +- **Worktrees:** `/home/badlogic/pi-observers` (docs, detach to upstream/main before each edit), `pi-s11` (slice 11), + `pi-sync-ex` (sync example, committed), `pi-s12`, `pi-s13` (committed slices), `pi-schema-switch`, + `pi-schema-chord`, `pi-schema-pkg` (schema work, committed; guard scripts untracked in pi-schema-switch, + `packages/coding-agent/scripts/schema-guard/`, owner never decided keep/drop), `p2-voice`, `p2-shots`, + `p2-schema` (private, committed), `/home/badlogic/pi-private` (private), `p2-live`, `pi-mono` (owner's). Phone deploys + only through the phone dev session `d602d653-61f1-4f99-aa54-99267e9dbf4e`, from owner-started turns. +- **Next after slice 11:** i18n (if decided), telemetry (slice 9), the small-examples pass; p3 work packages after the + 2026-11-03 meeting. +- **Process lesson (again):** never run a file write and the dependent git command in one parallel tool batch (it + happened 3 more times today). + +State as of 2026-10-28: **slices 6b and 8 are committed upstream** (pi `main`: 6b `14b4c56e5`, the design-audit fixes +for slices 1-6a `391e83fb1`, slice 8 `6eb99ad65`; CI green; merged into `private` `c684ee727`). Since 2026-10-27 all +of Chord v2 (`src`, its tests and `packages/chord/docs`, the decision log `decisions.md` included) lives in +pi upstream: commit there first (worktree `/home/badlogic/pi-observers`, detach to `upstream/main`), then merge into +`private`. p2's check is clean on `c684ee727`; the phone runs `ff7cdfeae` (p2 itself unchanged since). Next: slice 7 +(reload and rollout): spec written by this session reading the code itself, decisions one at a time, spec reviews, +build, reviews. Then slice 9 (telemetry; C11 `ctx.trace` to decide). + +**State 2026-10-29 (afternoon):** slice 7a (the dependency model) is committed upstream (`339847eb1`, CI green, merged +into `private`). Slice order now (owner): 7b reload in the kernel, 7c loader and dev, 10 HTTP and workers, 11 web and +TUI (the app kit, optional `chord/*` subpaths), 9 telemetry last. 7b's brief `docs/slice-7b/` (artifact +`slice-7b`, Radius `01m63r5q1febvtwpwqxrz76zn5`) and its six points are decided ("Slice 7b's open points, as +recommended"; plugin npm dependencies bundled, external by opt-in). **7b is building:** spec `slice-7b.md` +(`0c3675e64`), `s7b-builder` (radius/claude-opus-5-5) in `/home/badlogic/pi-7b`, brief `/tmp/s7b-builder-brief.md`, notes +`/tmp/s7b-build-notes.md`; then build reviews by `s8-spec-{astra,fable,opus,sol}` (Opus on radius), fixes, commit. +All Opus subagents run on `radius/claude-opus-5-5` (owner). The old `docs/explainer/` is stale (rewrite or delete: +owner). Dependency model docs: `docs/dependencies/`. + +**State 2026-10-30 (afternoon):** slice 7b is committed (`4b535d57e`), and so is the **simplification audit** +(`50af8c160`, CI green, merged into `private`): decisions A1-A15 in `decisions.md` ("Simplification audit"); cut +`installing` and the static registry cycle edges; retry by a hop budget (`retry.hops`); a fresh `bound` after a swap; +one rule after a live failure; `parked_long` for any long park (`state`); one seal token; about 255 code lines fewer. +Accepted noise: the text codec's small calls (+4-8%, code shape of its decoder, `benchmarks.md`). **Slice 7c (plugins) +spec** is written (`docs/slice-7c.md`, uncommitted in `/home/badlogic/pi-observers` with its decision "Slice 7c's +points" and design §21; patch backup `/tmp/7c-docs.patch`); spec reviews by Fable and Opus +(`/tmp/s7c-spec-review-{fable,opus}.md`) are folded only in part: the owner decided Q1 (b) a `mapStack(error)` helper, +Q2 a facet throwing while evaluating fails its host's whole step, Q4 copied files may not import their plugin's packages, +Q5 no kept esbuild contexts and remote shipping in 10/11 but the build result is plain data exposed for them; **open: +Q3, per plugin at startup and reload** (the owner wants it analysed: why all-or-nothing was chosen, history #66178, +#67234-#67252; my analysis says v2's closed schemas, version binding with waiting, ownership and registry versions remove +the old counter-traces; next step proposed: Opus and Fable punch holes, then the owner decides). The mechanical review +fixes (complete facet sets per host, relative source maps, host-package namespaces, `import.meta` as build errors, +watcher ignores, require order, discovery details) still need folding into the spec. Then: 7c build, 10, 11, 9. +`audit-builder` (radius/claude-opus-5-5) is idle with the v2 code in context. + +**State 2026-10-30 (evening):** **slice 7c (plugins) is committed upstream** (`a14cdf90f`, docs `b7e824952`): +`src/node/plugins/` (discovery, esbuild builds, vm evaluation, `pluginFile`, `mapStack`, `installDependencies`, +the watcher, `rollout`), `host.load`/`host.reload` per plugin with the closure rule, `details.committed` on an abort +after a reload's commit (owner "yes"), examples 35-39 as folders with real plugin directories (owner: full coverage), +benchmarks in `benchmarks.md`. Fix round 1 had 19 items from four build reviews and rechecks (all clean). The previous +paragraph's 7c state is superseded. Reading the sources (owner, 2026-10-30): understand the APIs (every +`src/*/README.md` and `index.ts`, `node/plugins/README.md`, the `Self`/`StoreSelf`/`Ctx`/`Host` interfaces); +read implementations in full only where this session or a subagent changes them (all of `src` is ~1.26 MB, more +than one context). Slice 10 (HTTP and workers): spec `slice-10.md` approved (owner: "just build it, have it reviewed, ensure we have +cute examples"); decisions in "Slice 10's points" (D1-D12, D10b, R1-R6, Q1-Q6, "The app kit is malleable by plugins"; +no automatic worker restarts, no route backoff timer); two spec review rounds folded (`/tmp/s10-spec-review*-*.md`). +**Building:** `audit-builder` in `/home/badlogic/pi-7b` (detached at upstream `bc9a20f84`), brief +`/tmp/s10-builder-brief.md`, notes `/tmp/s10-build-notes.md`; examples 40-43+ must be small and cute. Then build +reviews by `s8-spec-{astra,fable,opus,sol}`, fixes, commit upstream. Then 11, 9, the small-examples pass. + +**State 2026-10-31 (evening):** slice 10 (HTTP and workers) is built in `/home/badlogic/pi-7b` (uncommitted, base +upstream `bc9a20f84`), through fix rounds 1-3 (briefs `/tmp/s10-fix{1,2}-brief.md`, notes `/tmp/s10-fix1-notes.md`); +four build reviews, three security audits and two recheck rounds (`/tmp/s10-build-review-*.md`, +`/tmp/s10-security-*.md`, `/tmp/s10-recheck*-*.md`); Sol's second recheck found three leftovers (nodeRequest on TRACE +and userinfo Hosts, the authenticator chain after close, an aborted unread mounted `bytes` body leaking its scope) to +send to `audit-builder`; then fold spec and design §23, commit, CI, merge. Many decisions today (decision log, entries +dated 2026-10-31: origins only from relays, the HTTP security baseline with relays, the HTTP server as a native object, +a fetch core with Node as the easy path, settings are arguments, CSP nonce per response, API versions in import paths, +plugins import by name (to be reversed: see the plugin model proposal `docs/plugin-model/`), worker errors in +public). Open with the owner: the plugin model's six points, Chord's own frozen schema module replacing TypeBox (pi-ai +widening to accept both), slice 11 P3 onward (`slice-11.md` draft), the app kit design pass (`chord/app`, with +user management and worker-less shapes). Docs written today: `docs/http-workers/` (Radius +01m692mkyzekvr2advcnn9hb1g), `docs/guide-plugins-hosts/` (tutorial, Radius 01m693kb2re9aa54c51cynbsy1, v10), +`docs/plugin-model/` (Radius 01m69nqg5te8vrbhvat1vb2fkq). Explainers go to Radius by default (owner). + +**State 2026-10-31 (night):** **slice 10 is committed upstream** (`8959ad6e6`, fixes `cc08278a6` build without DOM +types, `20362c155` a CI heap bound; docs `b22c5f694`: design §23, `slice-10.md` as built, explainer version 2; CI +green; merged into `private`). Fix rounds 4-5 closed Sol's leftovers (`/tmp/s10-recheck3-sol.md` clean, 2,235 +tests). **Schema package** decided (`81f3cd412`, "A schema package beneath pi-ai and Chord"); measurement +`/tmp/schema-measure.md` (cfworker accurate but 16-330x slower than TypeBox's JIT; omptype weak on JSON Schema and +`new Function`; a closure-compiled prototype `/tmp/schema-measure/proto.mjs` matches TypeBox's speed): recommended +our own closure-compiled validator, 1.5-2.5k lines; open with the owner: write its design, and moving `Type` to +`pi-ai/typebox`. **Plugin model** open, explained to the owner as eight points (1 reverse import-by-name, 2 the app's +API with Chord re-exported and generations, 3 API split per host kind, 4 host kinds on registry tokens, 5 third-party +contracts, 6 `testPlugin`, 7 single-file plugins, 8 built-in plugins). All Claude subagents run on +`anthropic/claude-opus-5-5` (owner, 2026-10-31; replaces radius Opus/Fable). `schema-measure` (anthropic Opus) holds +the schema measurement context. + +**State 2026-11-01 (morning):** **TypeBox is gone from pi** except the coding agent (for extensions): pi-schema +(`@earendil-works/pi-schema`, its own closure-compiled validator, no eval; opt-in coercion `compile(s, { coerce: true +})` with `coerce`/`parse`, the superset of TypeBox's `Value.Convert` and pi-ai's old rules; `Static` also types +TypeBox-built schemas) is under pi-ai (`6536f8012`), pi-agent-core, pi-durable, Chord v1 and v2 (`be039fb31`, JSON and +fingerprints identical, `format` an annotation), benchmark vs ata/Ajv/Zod/TypeBox/cfworker/omptype (`a6bde34`, +Radius 01m69x3dz2fsg9k1r33vde7hv6), TypeBox `Static` fallback (`6291b9bcc`); decisions "A schema package beneath +pi-ai and Chord" and "The schema switch". Extensions guarded by `packages/coding-agent/scripts/schema-guard/` +(untracked, in `/home/badlogic/pi-schema-switch`; baselines `/tmp/schema-guard/`). p2 migrated in private +(`387404afc`, every schema's JSON unchanged), private pushed `23f7c90c1`. Next: phone redeploy when the owner asks +(through the phone dev session, owner-started turn only); then the plugin model (primer `docs/plugin-primer/`, +Radius 01m69waqspfwwt21bjhvsp5w28, eight open points). + +**State 2026-11-01 (afternoon):** the plugin model (points 1-8), the app kit (points 1-5) and slice 11 (P3-P9) are +decided (decision log, 2026-11-01) and folded into design §21 ("The plugin model") and §23 ("The app kit", "Web and +TUI"). Owner: "go ahead, just do all the things. tired of reviewing": this session writes specs, runs builds and +reviews and commits without asking per point (only truly new decisions go to him). Slice order now 12 (plugin model, +`slice-12.md`), 13 (app kit, `slice-13.md`), 11 (web and TUI on the kit, `slice-11.md` rewritten), 9 (telemetry). +**Slice 12 is building:** `audit-builder` (anthropic Opus) in `/home/badlogic/pi-s12`, brief `/tmp/s12-builder-brief.md`, +notes `/tmp/s12-build-notes.md`. Then reviews by `s8-spec-opus` and `s8-spec-sol`, fixes, commit, then 13, then 11. +Separately: `voice-doer` audits and fixes p2's voice mechanics in `/home/badlogic/p2-voice` (brief +`/tmp/voice-audit-brief.md`; owner's rule: VAD may misfire, only Parakeet's real text interrupts; stop work at once +when the user speaks; speak text before tool calls; choppy audio); deploy through the phone dev session when done. + +### Committed Chord v2 + +- L1, slices 1, 2, 3a, 3b, 4a, 4b, 5 (`4a5b45c45`), the simplification pass (`79a0206dc`), the watch adaptation + (`cc2d61f53`), 6a (`6bd526ebd`), 6b (`14b4c56e5`), audit fixes (`391e83fb1`), 8 (`6eb99ad65`). Order: 8 before 7 + (owner, 2026-10-27: reload has to reload real registries). Specs `slice-6b.md`, `slice-8.md` (built). + +### Decided 2026-10-26/27 (decision log, each entry folded into design.md and the spec) + +- Slice 6a's first-draft points: "The host's store and how it is opened", "A Chord transaction enters the commit step + as Durable's does", "The table API in `self.tx`", "Chunks of `rows` migrations" (100), "`lastSeq` instead of + \"head\"", "Export archives are operations, the encoding is pluggable", "Writes, commits, batches, archives", "The + outbox is optional, internal and versioned", "The `json(...)` column helper", "The commit-step benchmark", "Slice 6a's + three code questions" (ctx on keyed reads, the paused deadline, `client` host kind). +- General model (owner: "model discussion done"): "Facet documents are removed" (C16), "Facets only declare" (no + routines), "Documents and the line belong to the object, not the attempt", "Job runs belong to the object; + singletons stay resident", "The seal fence", "A live object can fail", "A native restart drains first", "Keyed + state members, `local` contracts only". Lazy creation, activation waiting for `self.get` dependencies and automatic + retry stay. Owner's rule: **Chord and Durable align as much as possible** (design §1.10). +- First spec review: "Writes carry their versions; archives are lossless", "Store errors: Durable's rule", + "`runCounted` in `chord/sqlite`" (pi `8025c414d`, verified in workerd: `rowsWritten` counts index writes), + "Restricted names instead of assigned table names", "Nullable columns and the `set` form", "The migration API: the + reviewers' consensus" (the owner delegates the shape), "Cold keyed loads on the line", "Slice 6a small items" (20, + among them: no hand-written memory backend, tests on SQLite `:memory:`). +- Second spec review: "Second spec review: the majority" (`evolveTable`, spec section 15). "State and job names are + restricted too", "Size is not a goal by itself" (no cut pass). Build reviews: "Slice 6a build reviews" (member + sources re-bound by name across attempts, the migration yield, counters). + +### Next + +1. Slice 7 (reload and rollout): spec (the slice 7 carry-over in `slices.md`, including slice 8's "Registries on + reload"), spec reviews, build, reviews to the bar. Builder model Opus 5.5; reviewers `s8-spec-{astra,fable,opus,sol}` + have slice 8's context. +2. Slice 9: telemetry. + +### Waiting and other + +- **Other:** the owner's own conversations: `ultimate` (Daniel's Ultimate), `issue-10959`, `p3-walkthrough`; leave. +- **Deferred decisions** (design §26): jobs as objects, where a flow lives, project plugins, a DO profile, p3 domain + points 17-23, #24-#26, #28, #29. +- **Process lessons (2026-10-26, revised 2026-10-27):** the bar is decision "The performance and memory bar, and + converging reviews" (CPU within about 5% of HEAD per hot path, memory no worse beyond noise, GC probes); reviewers + benchmark against a HEAD copy (bursts, alternating, pinned; allocation and retained memory). Re-reviews reuse the + same reviewers (restore them; archiving keeps their context); a fix round adds no new decisions or features. Never message the phone session from a turn not started by the owner + (it is refused). When a reviewer on an OpenAI model (Astra, Sol) fails with a model error during security work, it + was likely a refusal: send it a message that this is a defensive review of our own open-source code (pi, packages/chord), + run against local copies, then let it continue (owner, 2026-10-31). Never commit a file write and the dependent git commit in one parallel tool batch (it happened + three times: the commit missed the edit). Never call `history_handoff` unless the owner says so; the app may compact + on its own, so keep this section current (2026-10-27: it was stale at a compaction). + +## 4. Read, in this order + +1. This file. +2. `packages/chord/docs/decisions.md` (the decision log; later entries win). +3. `packages/chord/docs/design.md` (the one current statement of every rule) and `slices.md` (the slice plan, + the feature map and the carry-over). +4. The current slice's spec (`slice-6a.md`, draft; `slice-5.md` as built), `store/book.md` (the store design) and + `benchmarks.md`. +5. The code in `packages/chord/src/` (contracts, data, documents, state, host, jobs, link, remote), `test`, + Chord's shared `src/watch.ts` and `src/sqlite`, and Durable's `session/session.ts` and storages (for 6b). +6. Background when needed: `durable-observers/book.md`, `p3-reviews/`, `apps/p2/docs/research/`. + +## 5. The owner's rules (binding) + +- **Never decide for the owner.** Present problems with a concrete example and proposals; he decides. Small API shapes + too. Subagents' questions go to him, relayed with options and a recommendation. +- **One issue at a time.** Do not move on until he says so. When he says to finish a discussion first, do not spawn + subagents or start work. +- **Write every decision into the decision log immediately**, fold it into `design.md`, and mark every log entry it + supersedes ("**superseded in part** by …" or strike-through), all in the same commit. +- Explain with concrete code, data and short traces; prose alone is impenetrable. Short lines, phone-readable. +- Be honest; no improvising; no complexity to please. Reuse before writing (Delta, `state.ts`, context, output, + `chord/sqlite`, Durable). +- No development or test modes: one behavior everywhere (owner). +- Style: tagged unions always; no ceremony; no `any`; erasable TypeScript; top-level imports; no backward + compatibility unless asked. +- Design docs are written by this session, not by a subagent without context. +- **Carry-over (owner, 2026-10-22):** everything deferred to a later layer is recorded in + the section "Carry-over" of `packages/chord/docs/slices.md` under that slice, in the same change; each slice's spec includes its + section and the build checks it off. +- **Code style (owner, 2026-10-22):** all Chord v2 code follows the section "Chord v2 Code Style" in the root + AGENTS.md (in every agent's context); every code brief and every review names it. +- **Code subagents never commit or push unless told to** (owner, 2026-10-22): implement until done and report; reviewers + review; the subagent fixes; the owner reviews the uncommitted changes in p2's workspace tab; then this session tells it + to commit. New Chord v2 code lives in `packages/chord/src/` (pi upstream since 2026-10-27, `93029a0f1`). + +## 6. Environment and flows + +- slayer: repo `/home/badlogic/pi-private`, branch `private`; remotes `origin` (pi-private), `upstream` (pi). Node 24: + `/home/badlogic/node-v24.4.1-linux-x64/bin`. Commit docs with explicit paths; push `origin HEAD:private`; when a push + is rejected: `git fetch origin && git merge --no-edit origin/private` (never rebase or stash). +- **Everything outside `apps/` is pi first, Chord v2 and its docs included** (owner, 2026-10-27: Chord v2 and all + its docs, the decision log too, live in pi upstream: `packages/chord/src`, `test`, `docs`, + `docs/decisions.md`): worktree of `upstream/main` (`/home/badlogic/pi-observers`, `npm ci --ignore-scripts`, + `(cd packages/ai && npm run hydrate-model-data)`, `npm run check`, explicit test files only, never the full Durable + suite (paid e2e), push to `upstream main`, wait for green CI, then merge `upstream/main` into `private`). +- Chord v2 tests: from `packages/chord`, `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run + test`. Playground: `apps/chord-playground`. +- Subagents: none of this session's are active (all archived on 2026-10-26); `ultimate`, `issue-10959`, + `p3-walkthrough` are the owner's (leave). Worktrees: `/home/badlogic/p2-live` (runs the live p2 server, tmux `p2`; + keep); `/home/badlogic/pi-observers` (upstream pi worktree used for today's pi changes, detached at `6c384aa30`, + clean; reuse for the next pi change after `git fetch upstream && git checkout --detach upstream/main` and + `npm ci --ignore-scripts`). Durable/Chord tests without a build: vitest configs `/tmp/observers/vitest.durable.config.ts` + and `/tmp/observers/vitest.chord.config.ts` (alias pi-ai and pi-telemetry to src). The phone dev session `d602d653-61f1-4f99-aa54-99267e9dbf4e` deploys p2 + to the phone on request (only from owner-initiated turns). +- Radius artifacts: always `visibility=private` (mcporter usage in `/home/badlogic/AGENTS.md`). Explainer build: + `node /home/badlogic/.pi/p2/skills/explainer/build.mjs <md> -o <html>`; the store book and walkthrough are + published as private artifacts (book `01m5g0jztafkrvk6kq8w88h6hx`, walkthrough `01m5g0ms6demgtyyfdznhxnfre`). diff --git a/apps/p2/docs/p3-reviews/digest.md b/apps/p2/docs/p3-reviews/digest.md new file mode 100644 index 000000000..e103bdc18 --- /dev/null +++ b/apps/p2/docs/p3-reviews/digest.md @@ -0,0 +1,266 @@ +# Digest — native-object code-change detection review + +## Status and scope + +This records the findings, review points, questions and recommendations produced by the `digest` subagent in this conversation. No implementation changes were made. Experiments were written under `/tmp/digest-exp/`. The functionality map is not included. + +**Owner-supplied design assumptions (decided for this review):** + +- Natives (sockets, processes, databases, watchers, sandboxes and class instances wrapping them) may only be held by objects, through `self.open(async open, close)` in the object's factory. The value never changes during the object's life. +- Facets are pure objects and may not open natives. +- Factories are synchronous and declare only. +- A plugin reload loads the facet bundle using loader D (one esbuild script per facet), stages facet setup and every live object's new factory with a recording `self`, rolls back if staging throws, and then commits. +- Pure objects swap in place, with new methods and retained documents and references. +- Native objects restart (close, then reopen) only if their code changed. + +**Owner proposal, not a confirmed decision:** the native object's digest covers the module containing its factory (described as the `host.serve` call site) plus its transitive imports from esbuild's metafile. + +**Existing documented decisions used by the review:** loader D externalizes host packages and `apps/p2/src/**` into the base bundle; base changes cause full host restarts. Plugin graph guardrails prohibit dynamic imports, `require`, top-level await, module-level side effects and raw timers in facets. These are documented target rules, not claims that D is already implemented. + +**Still open:** all recommendations below, including `defineNative`, dedicated native modules, lint rules, `codeFiles`, digest manifests and native-holder separation. No owner acceptance of these recommendations was received in this conversation. + +The repository inspected still uses `self.resource` and resource versions. Its old resource replacement, dependency propagation and routines are evidence about existing code, not automatically decisions for the new `self.open` model. Where the original response assumed old resource-dependency behavior, that remains an open integration question. + +## Original report to the main agent + +The following preserves the original report, with its original assertions and estimates. Qualifications and additional review notes follow it. + +**Recommendation:** use a digest per native module, not per facet. Each native object type is declared in its own module, `plugins/<p>/native/<name>.ts`, with `export const X = defineNative(Type, factory, opts)`, and the facet only calls `host.serve(X)`. The digest is that module's import closure taken from the esbuild metafile. The proposal as written (the module with the `host.serve` call site plus its imports) is correct today, but in every real plugin it restarts on any facet change. + +### 1. Mechanics (measured) + +I bundled real facets in `/tmp/digest-exp/build.mjs`. `src/**` and bare packages were external. I used Node 24 because Node 26 on slayer is missing `libatomic`. + +- **The data:** `metafile.inputs[path].imports[]` gives `{path, kind, external}`. Internal edges form the graph, and a module's closure is a depth-first walk from it. The digest is sha256 over the sorted `(path, contentHash)` pairs of that closure. +- **Type-only imports are erased.** `radius/contract.ts` does `import type { RadiusBrain }` and has no edge in the metafile, so type-only imports never cause a restart. +- **Build cost:** radius/server.ts 15.8 ms, terminals/server.ts 7.0 ms, codemode/worker.ts 5.5 ms. Loader D runs this build anyway. +- **Closure and hashing cost:** 0.2–0.5 ms per facet. +- **Closures today:** + - `radius/server.ts`: 5 of 5 bundled files (brain, relay-access, contract, ports/contract). + - `terminals/server.ts`: 6 of 6 (including `sessions/contract.ts`). + - `codemode/worker.ts`: 4 of 4 (including `models/contract.ts`). + + The factory's module is the facet entry, so its closure is the whole facet bundle. Here the proposal is the same as the coarse rule. + +**Where the digest is stored:** +- The server computes it at build time and ships it in the facet manifest: `{facetDigest, natives: {modulePath: digest}}`. The revision already lists every facet with its digest (target-design §15). +- `defineNative` gets its module path injected by the bundler, the same way loader D injects `import.meta.url` (B4). +- Each host keeps the digest a live native object was opened with in memory. Natives do not survive a host restart, so nothing needs persisting. +- In staging, the recording `self` marks which objects call `self.open`. Commit restarts those whose recorded digest is not the new one. + +**Base bundle:** exclude host packages and `src/**`. A base change already restarts the affected hosts (§14.2). Do include the loader version and esbuild options, because a different build changes code without changing any source. + +### 2. Correctness: how behaviour can change without the digest changing + +| Way it can change | Matters? | Digest | Prevention | +|---|---|---|---| +| Closure over facet-level variables or functions | yes | caught only if the factory's module is the facet, which is coarse | `defineNative` must be top-level in a `native/` module; lint forbids `self.open` outside `native/` | +| Values passed in from the facet (`serveRadius(host, {onStatus})`) | yes | missed by a declaration-module digest | `defineNative` takes no facet arguments, and the facet calls `host.serve(X)` with nothing else (lint) | +| Registries and host values | only if captured in `open` | not code | read them at use through readables; capturing an entry in `open` keeps it stale. Hard to lint, so document it | +| References to other objects (`self.get`) | no | n/a | they are contract-versioned calls; deps already reopen on change | +| Dynamic `import()` | yes | string literals appear in the metafile as `dynamic-import`; computed paths are missed | already forbidden by the §14.5 static check | +| Shared helpers and cross-plugin files | yes | caught (they are bundled) | false positives only | +| Bundled `node_modules` (global plugins) | yes | caught; hashes cached by mtime and size | — | +| External host packages and `src/**` | yes | not counted | covered by the base restart | +| JSON and `?url` assets | yes | caught (they are metafile inputs) | — | +| Files read or spawned via `import.meta.url` | yes | **missed** | see below | +| Runtime config (`radius.json`, `P2_RADIUS`, env file) | data, not code | n/a | watch it, or use `retryOn`/deps | +| Module-level state in a native module | yes | n/a | lint: no top-level mutable state in `native/` | + +**Out-of-bundle files:** `terminals/detached.ts` spawns `main.ts` and the ghostty wasm, which never enter the server graph. It already keeps its own hand-written list in `terminalHostVersion()`, and that list can drift. Generate it from a metafile instead, or let `defineNative({ codeFiles })` add those files to the digest. + +**Module-level state:** a kept native keeps running old-generation code. After a reload, the bundle is re-evaluated, so new pure code sees fresh module state while the native's callbacks see the old state. An example is `models/worker.ts` `settingsSources` (a top-level WeakMap). The same split breaks `instanceof` against a kept native's class, so lint that too. + +**GC note:** esbuild puts all of a facet's modules in one scope. A kept native therefore pins its opening generation's whole script, but only that one generation, so the alive set does not grow over reloads. The leak detector should expect it. + +### 3. Granularity: plugins and how often digests would change + +Commit counts are for the facet closure versus the hypothetical native-only closure. Caveat: the git history covers only Oct 9–16 (65 commits, mostly a migration). The ratio is the useful signal. + +| Plugin | Native module(s) | Facet vs native commits | Cost of a restart | +|---|---|---|---| +| radius | `native/connection.ts`: `RadiusBrain` + `RelayAccess` (brain.ts, relay-access.ts). The `RequestResolvers` logic moves to a pure object | 11 vs 4 | drops the relay WebSocket, including the phone's own connection | +| terminals | `native/service.ts` (service, detached, protocol) plus the per-session view lease; `codeFiles` = the `terminalHostVersion` list | 14 vs 5 | terminal WebSockets drop; detached hosts survive | +| codemode | `native/sandbox.ts` (`CodemodeSandbox` is a host package) + calls.ts. Extension wiring stays pure | 11 vs 1 | kills running scripts | +| workspace (worker) | `native/watch.ts`: only the watch. `writer` is plain state; `viewer` is a liveness lease | environment.ts (890 lines) changed 6×; split watch from file logic | cheap re-watch | +| envs | `native/store.ts` (Observer) and `native/ssh.ts` (manager, remote-env, config) | 15 vs 2 | SSH connections close | +| models (server + worker) | today `openResources` pulls in model-config and provider-config parsing | 16 vs 9 (`model-config.ts` alone changed 18× since April) | low (file watcher) | +| search | `native/db.ts` (SQLite) | 13 vs 3 | trivial | +| login | `native/logins.ts` | `logins.ts` 4 | aborts an in-progress login | +| logs | `captureProcessLogs` + timer. The `collect` logic now inside `open` becomes a routine | n/a | cheap | + +For models, the rule should be: the native is only the Observer, and parsing becomes a pure derive. That would cut its digest changes to about 1–2. The same pattern applies to logs. + +**Layout rules that make the digest useful:** +- Natives stay thin: `open`, `close` and `expose`. +- All logic goes in pure objects that `self.get` the native. +- No other plugins' contracts in a native closure unless it needs them. + +### 4. Alternatives + +| Option | Correctness | Simplicity | Connection drops while iterating | +|---|---|---|---| +| Call-site module closure (the proposal) | good | needs a call-site locator (stack traces or an AST scan) | every facet edit, so the same as coarse today | +| **Declaration module closure + lints** | good; gaps closed by lint, plus `codeFiles` | one helper, one manifest field | only real native-code edits | +| Author-bumped `codeVersion` | bad: a forgotten bump means you reload and still run old code, with no signal | simple | minimal, but untrustworthy | +| Separate natives facet per plugin | same as mine, but all natives of a plugin together | needs a second facet per surface and changes discovery | envs store+ssh and codemode sandbox+calls restart together | +| Factory source text + imports | misses helpers and mutable module state; needs free-variable analysis (re-implementing the bundler) | complex | — | +| Coarse: restart all natives on any reload | correct except out-of-bundle files | simplest | worst: radius drops the phone and codemode kills the script doing the reload | + +The rule I recommend has the same cost as the proposal, closes the closure and argument gaps by construction rather than by luck, and cuts restarts about 3–10× for radius, codemode, search and envs. For models and logs the gain comes only after their parsing and polling move out of `open`. + +## Additional findings and reasoning produced during the investigation + +### Correctness and layout — recommendations still open + +- The native-holder object is the restart unit, not merely its socket or its `open` function. If resolver, extension or business logic shares its factory, an edit to that logic restarts the holder as well. Keep native-holder objects small and isolate behavior in pure objects. +- Radius concretely mixes `RadiusBrain`/`RelayAccess` ownership with the `RequestResolvers` contribution. Moving only `brain.ts` is insufficient if the digest still roots at `radius/server.ts`. Put the native declaration in a separate module and put resolver registration in a pure object. +- Codemode similarly mixes sandbox ownership, `RunningCalls`, settings-derived extension enablement and `Extensions` registration. Extension wiring should be replaceable without closing the sandbox. +- A helper such as `serveRadius(host, options)` does not solve closure tracking: facet callbacks or other values passed through `options` can change while the helper's module graph does not. A predeclared native descriptor with no arbitrary facet arguments avoids this hidden dependency. +- A callback created inside the native factory or `open` is covered by that factory's module and imports. The problematic callback is one injected from outside that graph, or fetched from a registry and then retained. +- A registry entry read at open-time can retain another plugin's old function after that plugin changes, without changing the native's digest. Read current registry values at use-time instead. This is a dataflow/lifetime issue, not something the import graph proves absent. +- Host-provided code is trusted base code for this purpose, but changing runtime host values is separate from changing host code. The digest does not establish what runtime values were used to open a connection. +- Bundled shared helpers must remain included. Excluding cross-plugin contracts to reduce churn is unsafe without proving those modules contain only inert declarations: contracts can include real runtime logic. Type-only imports are the safe way to avoid a runtime edge when only types are needed. +- `terminals/server.ts` currently pulls `sessions/contract.ts` into the native digest through Catalog use. A separate service-holder module can avoid some such irrelevant edges; a session-specific holder may still genuinely need Catalog. +- For JSON, text, wasm and other assets, inclusion depends on how the loader resolves and loads them. Assets supplied as esbuild inputs can be hashed; arbitrary runtime filesystem reads cannot. `?url` is a loader convention, not itself an esbuild guarantee. +- Computed imports, `require` and reflective code loading should be rejected rather than guessed at. Loader D already proposes these graph restrictions. +- Runtime configuration is not native code. `radius.json`, `process.env.P2_RADIUS`, env configuration and model settings need their own observation/reconciliation policy. A file digest cannot say whether a changed URL, credential or provider setting should reopen a native. +- The native module and its helper graph should have no mutable module-level state. A kept native runs closures from its opening generation while newly evaluated code has fresh module state; this can silently split caches, callback tables or identity checks. +- The cross-plugin `instanceof` warning also applies within one plugin across reload generations: a native created with the old class is not an instance of the newly evaluated constructor. Treat native references as opaque/structural values rather than testing them against reload-local classes. + +### Granularity and false positives — findings and open recommendations + +- A module digest is conservative, not semantic equivalence checking. Comments, unrelated exports and unused code in a hashed file can cause restarts even when behavior is unchanged. +- Hashing a whole facet's generated output would introduce further instability from bundler naming and ordering. The investigation favored original module content hashes, with accepted file-level false positives. +- Esbuild's metafile does not give per-factory compiled code or the lexical free-variable graph needed for `factory.toString()` plus imports. Function source text omits helper implementations, captured values and class behavior. +- A separate native facet per plugin is an organizational alternative, but facets themselves still cannot open natives under the owner-supplied rule. Such a facet would declare native objects. It also requires deciding discovery and surface naming, and restarts unrelated native objects together. +- Workspace's writer resource currently contains a WeakMap of workspaces and a serializing tail promise. This is not simply a filesystem watcher; deciding which parts are ordinary object state versus native wrappers is still necessary. The viewer resource is a liveness lease, not a socket or database. +- `workspace/environment.ts` mixes read/write operations and watching in 890 lines. Leaving it whole means any file-operation edit changes the watch digest; a narrow watch module is useful only if it does not import the large mixed module anyway. +- Envs has two natural native boundaries: its file observer (`store.ts`) and SSH/browser manager (`manager.ts`, `remote-env.ts`, config helpers). Their lifetimes and change frequencies differ, so a single plugin-wide native digest is needlessly coarse. +- Models has server and worker resource observers. `resources.ts` imports credentials, model store, model configuration and provider configuration. Merely moving the factory to `native/resources.ts` leaves all this parsing in the closure. Separating observation from parsing is the actual granularity change; the predicted 1–2 changes is an estimate, not a measured refactor. +- Search's `SearchDb` wraps `node:sqlite` and is already a small, isolated implementation module. Reopening it is much cheaper than dropping Radius or codemode. +- Login's `Logins` owns provider login flows. Its dependency on models resources is type-only in `logins.ts`, but the native is created with a live `config.resources` value. Its behavior and lifetime therefore have runtime dependency concerns even when there is no static import edge. +- Logs currently mixes process capture, native timer ownership, snapshot validation, deduplication and file polling in `open`. Separate process capture from collection behavior if retaining the capture across logic edits matters. +- Restart costs differ sharply: Radius disconnects the relay and relayed clients; codemode ends VMs/worker threads and running scripts; terminals disconnect views but do not kill detached host PTYs; SSH-manager restarts close connections; workspace watches, SQLite handles, config observers and log capture are relatively cheap to reopen; login restarts cancel in-progress authentication. + +### Terminal out-of-graph evidence + +`plugins/terminals/detached.ts` computes `terminalHostVersion()` using this explicit list: + +```text +main.ts +log-preload.ts +../../src/runtime/logs.ts +daemon.ts +protocol.ts +detached.ts +envs.ts +ghostty.ts +../../vendor/ghostty/ghostty-vt.wasm +``` + +It reads these files with `new URL(file, import.meta.url)`. It spawns a process using `../../src/source.ts`, `log-preload.ts` and `main.ts`. These process-entry dependencies are not import edges from the server facet. `service.ts` calls `terminalHostVersion()` while opening a `TerminalView`. + +The existing hand-maintained version list is precedent for explicit code dependencies, but its completeness was not audited. A separate esbuild entry for the spawned process, plus explicit binary assets as needed, was suggested as an alternative to maintaining a second hand-written graph. Whether a terminal-holder restart should automatically replace an outdated detached host is a separate policy; detecting a change does not by itself update that other process. + +### Metafile, cache and registration mechanics — open implementation details + +- Build the import graph once per facet; walk it from each native root. Hash shared module contents once and reuse them. For larger graphs, cache closures or collapse import cycles rather than rereading the same files per object type. +- The experiment used a visited set, so import cycles terminate. Cost is proportional to graph traversal and file bytes hashed; the reported sub-millisecond times are for small graphs, not a package-heavy benchmark. +- Bare host specifiers and externalized `src/**` appear in `imports` with `external: true`, but their implementation files are absent from `inputs`. Their implementation cannot be recovered by traversing the facet metafile. +- Bundled non-host registry packages should be included naturally in `inputs`. Their code changes count. The experiment externalized all bare specifiers as a simplification because the three chosen facets use host packages and builtins; it did not validate a real global plugin with bundled npm dependencies. +- A content hash cache keyed by mtime/size was proposed as a performance optimization. The correctness requirement is still a hash of the bytes actually built; mtime/size alone is not proof of identical content. +- Include build semantics in the digest: at least the bundler/loader version and relevant esbuild options such as defines and target. Source-only hashes miss transformations that change emitted behavior. +- Paths should be stable for the digest rather than checkout-dependent absolute paths. The experiment used metafile paths relative to the repository working directory; canonicalization and relocation rules were not implemented. +- A declaration module is easier to identify reliably than a `host.serve` call site or an arbitrary function's source module. A bundler transform can inject its stable module ID into `defineNative`; stack traces and runtime function inspection were rejected as fragile registration mechanisms. +- Native descriptors should be immutable declarations rather than factories that a facet parameterizes or wraps. Otherwise the source-module ID no longer captures the code or values actually installed. +- Store digest metadata on the declaration/type manifest and the opened object's host record. Do not infer it from function identity across bundle evaluations; every evaluation creates new functions even for unchanged source. +- Excluding base code is valid only because loader D's base changes restart the host. If base hot reload is introduced, include a base revision/digest or define equivalent invalidation. + +## Experimental evidence and limitations + +### Environment and graph output + +- Workspace: `/home/badlogic/pi-private/apps/p2` on slayer. +- Installed esbuild version observed: `0.28.2`. +- Script: `/tmp/digest-exp/build.mjs`; it used `bundle: true`, `write: false`, `format: "cjs"`, `platform: "node"` and `metafile: true`. +- Node was absent from the default PATH. `/home/badlogic/node-v26.11.1-linux-x64/bin/node` failed with `libatomic.so.1: cannot open shared object file`. `/home/badlogic/node-v24.4.1-linux-x64/bin/node` worked (`v24.4.1`). +- The script built radius/server.ts, terminals/server.ts and codemode/worker.ts sequentially. Timings are one run, not statistically controlled measurements or an implementation of the complete loader D wrapper. + +Internal graph, with external leaves omitted: + +```text +radius/server.ts + ports/contract.ts + radius/brain.ts + radius/relay-access.ts + radius/contract.ts + radius/relay-access.ts + +terminals/server.ts + sessions/contract.ts + terminals/contract.ts + terminals/service.ts + terminals/detached.ts + terminals/protocol.ts + terminals/protocol.ts + +codemode/worker.ts + models/contract.ts + codemode/calls.ts + codemode/contract.ts +``` + +- `radius/brain.ts` alone had a 2-file internal closure; `terminals/service.ts` had a 3-file closure; `terminals/detached.ts` had a 2-file closure; `codemode/calls.ts` had a 1-file closure. +- Radius external leaves included pi-ai Radius provider/config, `ws`, Node builtins, runtime Chord/server/logs modules, Chord and TypeBox. +- Terminals external leaves included Chord and Chord/node, `ws`, Node builtins, runtime Chord/principals/server modules and TypeBox. +- Codemode external leaves included pi-codemode, Durable/codemode, Durable, Chord/context, runtime worker/durable/Chord modules, Chord and TypeBox. +- The metafile demonstrates static runtime edges and erased type-only imports. It does not establish behavior equivalence, runtime closure freshness, successful native preservation, close/open ordering or GC behavior. + +### History evidence + +The first history query used `git log --follow` per file; the later aggregate query used ordinary path history without `--follow`. Those are different measurements and should not be mixed. For example, `radius/brain.ts` had 8 followed-history touches, while the ordinary-history union for brain plus relay-access was 4. The original report's aggregate counts were the latter, not a complete historical simulation of restarts. + +Individual followed-history counts produced during the investigation: + +| Files | Counts | +|---|---| +| radius/server.ts, brain.ts, relay-access.ts, contract.ts | 7, 8, 4, 2 | +| terminals/server.ts, service.ts, detached.ts, protocol.ts, contract.ts, main.ts | 7, 2, 3, 3, 4, 7 | +| codemode/worker.ts, calls.ts, contract.ts | 7, 1, 4 | +| workspace/worker.ts, environment.ts, protocol.ts, contract.ts | 7, 6, 4, 7 | +| envs/server.ts, manager.ts, store.ts, config.ts, remote-env.ts, schema.ts, contract.ts | 9, 3, 1, 2, 3, 1, 5 | +| models/server.ts, worker.ts, config.ts, resources.ts, provider-config.ts, model-config.ts, credentials.ts, model-store.ts, contract.ts | 7, 5, 3, 2, 2, 18, 2, 2, 8 | +| search/server.ts, db.ts, contract.ts | 7, 3, 7 | +| login/server.ts, logins.ts, contract.ts | 8, 4, 6 | +| logs/server.ts, contract.ts | 7, 4 | +| sessions/contract.ts, ports/contract.ts | 9, 5 | + +These were queried since April 1, 2026. The non-follow plugin history observed ran from October 9 to October 16, 2026. These counts primarily demonstrate facade/migration churn versus some more stable implementation files; they do not establish an expected future connection-drop frequency. + +The aggregate radius, terminals and codemode sets match the measured metafile graphs. Other aggregate sets were hand-selected approximations, not generated full closures, so the original table's 13/3, 15/2 and 16/9 values should not be treated as exact end-to-end digest simulations. + +## Qualifications and open questions + +These qualify claims in the original report rather than recording new owner decisions. + +1. **Static code versus all behavior.** “Correct today” is too broad without the runtime-value and out-of-graph qualifications. A metafile digest detects changes to included code, not every possible behavior change. No source-only rule can account for mutable runtime inputs without an additional lifecycle policy. +2. **Factory location versus registration location.** The question conflates the factory's containing module with the `host.serve` call-site module. Once a factory is imported, these differ. The recommendation intentionally roots at a declared native module, not the importing facet. The API and bundler transform remain unimplemented. +3. **Native-reference dependencies.** The original table says “deps already reopen on change,” based on old `self.resource` semantics. The new model says natives restart only for code changes and their values never change during object life. It remains open how a native holder depending on another native holder is invalidated when the provider restarts, and how old native values or callbacks are prevented from being retained. Contract-versioned method calls do not by themselves solve a captured `config.resources` value or terminal view. +4. **Registry capture lint.** Directly banning known APIs at `open` sites is possible; proving no registry function or host value is indirectly captured is harder. The recommendation does not supply a sound dataflow lint implementation. Decide whether a documented rule is sufficient or whether the native API must restrict such inputs structurally. +5. **Lint reach.** The proposed restrictions need to apply to helper modules in the native closure, not just to the declaration file; otherwise mutable module state or hidden loading can be moved into a helper. +6. **Generation retention.** The original report reasoned that a kept native can retain its old facet generation because esbuild scope-hoists modules. No heap or GC experiment proved that every retained native necessarily pins the whole script, nor that exactly one old generation remains for an entire facet. Several native types can retain different opening generations. This must be measured and reflected in generation-GC tests and the leak detector, not assumed away. +7. **Kept native behavior.** A native wrapping a class instance contains old-generation methods even if new pure objects swap methods. The no-mutable-module-state and no-cross-generation-`instanceof` rules address part of this, but the boundary between thin native wrapper behavior and reloadable pure behavior still needs specification. +8. **Timers, leases and mutable wrappers.** Workspace and logs expose ambiguity about what counts as a native under the new model. Existing resources sometimes hold mutable containers, promises, callbacks, liveness leases or wrappers that replace internal watches during relocation. Their mapping to immutable-lifetime `self.open` values has not been designed here. +9. **Runtime config lifecycle.** `retryOn`, deps and routines are mechanisms from the inspected target docs, not confirmed APIs in the owner's new model. Decide how config-driven reopening happens without mislabeling data changes as code changes. +10. **Assets and separate processes.** Decide between explicit `codeFiles`, separate build entries and another asset-dependency mechanism. A main.ts build may still need explicit wasm/native binary inputs. A digest change in a client-side holder does not guarantee a detached process has adopted new code. +11. **Hash fidelity.** The experiment reread files from disk after esbuild completed. A production implementation should hash the exact inputs used by the build, avoiding an edit-between-build-and-hash race. Loader D build options, source rewrites, resolver choices and injected paths must be covered by the digest scheme. +12. **Manifest identity.** The original sketch keys native digests by module path. One native type per declaration file makes that workable; otherwise metadata also needs a stable declaration/type identity. The host must retain the digest of the code that actually opened the kept native, not merely overwrite it with a newer facet revision. +13. **Rollback boundary.** Staging throws before commit are covered by the owner-supplied rollback rule. Post-commit close/open failures are not made transactional by a digest. This investigation did not settle their readiness, failure, retry or reporting semantics. +14. **How much false-positive reduction is worth it?** The 3–10× claim is illustrative history-based evidence, not a benchmark of a completed split. Native-heavy subsystems such as models can keep changing frequently unless dependencies are reorganized, and comment-only changes still restart them. + +## Recommendation, still awaiting owner decision + +Adopt **a dedicated declaration module per native object type, with a digest of that module's bundled static import closure**, plus explicit coverage of executable/asset files outside that graph. Keep facets declarative and native holders thin. Do not permit facet-injected callbacks or arbitrary values to silently parameterize an otherwise unchanged native declaration. Keep host/base code external only while base changes require host restart. Store the build-produced digest with the native declaration and the digest actually opened with each live object. + +Use static rules to prevent the straightforward hidden dependencies (nested/parameterized declarations, native opens outside allowed modules, dynamic loading and mutable module state), but do not claim lint or a metafile proves runtime behavior equivalence. Specify runtime config changes, provider-native replacement, cross-generation identity and retained-generation accounting separately. The coarse rule is simpler but drops Radius and codemode during unrelated iteration; manual code versions are easy to forget; function-source hashing is not a reliable substitute for an explicit code boundary. diff --git a/apps/p2/docs/p3-reviews/dsh-compare.md b/apps/p2/docs/p3-reviews/dsh-compare.md new file mode 100644 index 000000000..296e68d1d --- /dev/null +++ b/apps/p2/docs/p3-reviews/dsh-compare.md @@ -0,0 +1,208 @@ +# dsh-compare — DeepSeek Harness comparison research subagent + +## Status and scope + +This archives the findings, review points, open questions and recommendations produced earlier in this conversation. It is not a new owner-approved design. The report below is preserved verbatim where possible; supplementary observations from the research commentary follow it. Evidence paths refer to `/home/badlogic/research/deepseek-harness` at `d743267388641bc76f17c45ce8b4c231aed1d32c` unless otherwise stated. Chord comparisons refer to `origin/private:apps/p2/docs/target-design.md`, `plugin-loading.md` and `questions.md` as consulted during the research. + +**Owner-decided requirements known from the supplied target design:** in-place plugin reload on every host without restarting it; remote objects with principals; plugins working across p2 versions; running calls and jobs finish with old code rather than being drained by reload; same-version resources kept; resource replacement/handover rather than an object restart; immutable pinned registry tables; no cross-host transaction; base/runtime code changes restart affected hosts. The target explicitly rejects drain-based handover, object restart for state changes, late-bound registry entries and `restart_required`. The owner's assessment supplied in the task was that objects plus references have no issues or footguns, whereas the other concepts—especially natives—do. + +**Still open / recommendations only:** every proposal below to remove `self.resource`, move natives into base supervisors, restart native-owning objects, make host installation all-or-nothing, or change Questions semantics. No owner acceptance of those proposals occurred in this conversation. Some intentionally conflict with the recorded target and would require a new owner decision. No later p3 decisions are known to this subagent. + +## Original report + +**DeepSeek Harness (DSH) vs Chord: natives, reload, state** + +Read-only research; I made no edits. DSH checkout HEAD is `d743267` (2026-10-09). DSH's `native/` directory holds native addons (Landlock, flock) and is unrelated to "natives" in our sense. + +### 1. Natives and their lifecycle + +DSH runs on Cordis, a plugin framework. In Cordis everything a plugin does is an *effect* inside a *fiber* (one running plugin instance). When the fiber is disposed, its effects are undone in reverse order and awaited until they have actually stopped (docs/cordis-primer.md:13, docs/defensive-patterns.md:19). + +Natives follow one pattern throughout DSH, though it is a convention and nothing enforces it: + +- **A few long-lived services own the natives:** processes in `ctx.subprocess`, PTYs in `ctx.terminals`, background work in `ctx.jobs`, plus LSP and MCP connections and session persistence. Ordinary plugins such as tools hold opaque branded ids and call methods. "The service owns process coordinates and lifetime; consumers own what a process means" (packages/subprocess/subprocess/README.md:98). +- **A native's lifetime belongs to its owner, not to the code that created it.** Spawning takes the exact owning `Agent`. The service then attaches its cleanup to that agent's scope: `owner.ctx.effect(...)` in packages/terminal/terminal/src/index.ts:322-332 and packages/jobs/jobs-local/src/index.ts:610-627. "Registrations outlive producer and controller fibers" (packages/jobs/jobs/README.md:73). +- **Access is fenced by the owner.** Another agent gets `FOREIGN_SESSION` (packages/terminal/terminal/README.md:52-56). +- **Reconnecting lives inside the owner.** The MCP client has a connection supervisor that keeps numbered client generations, retries with backoff and rebuilds its registrations; consumers only see tools come and go (packages/mcp/mcp-client/README.md:93, :118, :129). +- **Execution world:** the filesystem and subprocess services are swapped together for remote providers. Handles expose no path or pid (.agents/notes/implemented/architecture/2026-09-11-subprocess-control-pipe.md:21). + +**The generalization we lack:** natives are never values in plugin code. They live in owner services, are reached by id, and each one is bound to an owner identity. A native's lifetime is the shorter of the owning service's fiber and the owner's scope. A consumer reload keeps natives, because the owner's code did not change (subprocess README:113). A reload of the owning service kills them. + +### 2. What DSH constrains that we allow + +- **No native survives a code change of its owner.** There is no `replace`, no resource version and no handover. +- **Dependents always restart.** A fiber's activation epoch is the list of its providers' fiber uids (vendor/cordis/src/fiber.ts:611-622). A provider reload changes that list, so every consumer is torn down and set up again (fiber.ts:625-640). Their own note calls it "correct, if heavy" (2026-07-23-client-plugin-loading-model.md:95). +- **No in-memory state survives a reload.** The fiber is fresh, and so is React state (same note, :97). State that must last goes into the session event log or the storage domains. +- **No mixed versions.** A reload batch is all-or-nothing: if activation fails, the old modules and plugins are restored (packages/boot/hmr/src/index.ts:678-720). +- **One Node host process,** plus browser clients that call `@Remote` methods. Ids are resolved to host objects (`agentId` → `Agent`) by lookups (docs/api-gateway.md:11). There are no remote objects and no principals beyond the owner fence. +- **`restart-required` exists:** package version changes and bootstrap modules need a restart (packages/boot/plugin-manager/README.md:52, :140). + +What this buys them: none of our replace ordering, `deps` handover, stale handles, "read `.value` at use", derive rebinding or rollout coverage. Those cases simply never happen. + +### 3. Code changes and reload + +DSH does reload in place (Cordis HMR on the host, a `client-hmr` plugin in the browser). It does not swap code under live state. A reload means: + +1. Dispose the plugin's fibers and wait for them to settle (hmr/src/index.ts:686-687). +2. Plug in the new module with the same config, which cascades to every dependent. +3. On failure, roll back. + +The costs are accepted openly: + +- Reloading the agent-loop provider "stops and drains every live handle it made" (packages/core/agent/README.md:117). Sessions are later resumed from the log, with interrupted turns closed. +- Reloading the persistence backend makes writes "fail loudly until the sessions restart" (2026-08-27-handle-based-session-persistence.md:35). +- Reloading the terminal service closes every terminal (terminal README:81). +- Running job and tool closures keep their old code until they settle. This is the same as our jobs. + +### 4. State, persistence, background work, prompts + +- **State:** + - The append-only session log is the only durable truth; projections are derived from it (docs/architecture.md:118-126). + - Non-session data lives in versioned storage domains (packages/storage/README.md:12). + - Format changes are adjacent migrations that never overwrite a committed generation. +- **Background work:** + - Jobs are in-process and "die with the harness process" (jobs README:55). The README admits the contract cannot go cross-process (:126). + - Schedules are durable records. After downtime only the latest due occurrence is delivered (docs/subsystems/schedule.md:146). +- **Automatic work does not resume by itself.** Goal "activation" is kept out of replay, so resume or fork needs a human action first (docs/glossary.md:27). +- **Questions and approvals:** + - While the tool call is open, the call itself is the question. After the call returned or the process died, the question is "continued" and an answer comes in as a new message (packages/interaction/user-questions/README.md:35). + - No pending promise needs to survive anything. + - Approval fails closed: `unavailable` counts as a denial (docs/subsystems/approval.md:20). + +### Comparison + +| Topic | Chord target | DSH | +|---|---|---| +| Native holder | resource in any object | owner service only | +| Plugin holds | `.value` (the native) | id plus methods | +| Native owner | the object that opened it | explicit `Agent` scope | +| Code change | keep / replace / reopen | restart | +| Dependents | `deps` graph | restart (epoch) | +| Old code | kept natives, stale handles | only in-flight closures | +| Mixed versions | per facet | none (rollback) | +| Reconnect | routine | inside the supervisor | +| Pending question | resource store | open call, else a message | +| Hosts | many, rollout | one process | +| Gives up | nothing (complex) | kills live state on provider reload | + +### 5. Which ideas remove which of our problems + +- **Natives only in owner services; plugins hold ids.** Removes "old code inside kept natives" and "natives shared across plugins": sharing is a served type, which is already our rule. It also removes the reviewers' worry about plugin class instances holding behaviour: those are code, rebuilt on reload, not natives. +- **Explicit owner at spawn** (`spawn(spec, { owner })`). Removes "natives born in requests": a request-born native belongs to the calling object identity or to a job, never to a code generation. It also gives keep-alive a single source: the host counts live natives per owner (DSH: `hasOwnerActivity`, terminal README:85; the jobs registry reporting session activity, jobs README:83). +- **Supervisor with generations.** Removes "reconnecting natives" from plugin code; consumers read a status readable. +- **Restart instead of replace.** Removes "replace and dependents" entirely. In our world the cascade already exists: a reference ends with an error and is re-held (§6). Dependents become references, the one part the owner calls footgun-free, instead of `deps`. +- **All-or-nothing per host with rollback.** Removes "mixed versions within a host". +- **Owner services in runtime code.** If the generic supervisors (process, PTY, socket or connection, SQLite) live in the base bundle as host values, a plugin reload never touches a native, and a change to a supervisor is a host restart, as base changes already are. +- **Question = open call; after loss = input.** Removes the Questions store as a kept resource with `replace: old => old`. +- **Registry readiness:** Cordis keeps consumers pending until their injected services exist, and does a startup-only audit (2026-09-09-consumer-owned-startup-strictness.md:19). This roughly matches our start barrier; no new idea here. Their registry entries are reversible effects that go away and come back on reload, so there is a gap, which we deliberately avoided. +- **Durable phases across reloads:** DSH freezes each request and kills live agents on a core reload. Our immutable pinned registry tables are already better here; keep them. + +**Ideas that don't fit p2:** + +- **Cordis's in-process cascade** cannot cross hosts or principals. Our reference error plus re-hold is the remote form of the same thing. +- **Killing live agents on reload** conflicts with "live reload on every host". The fix is to keep natives out of plugins, not to accept the kill. +- **Owner fencing by `Agent` object identity** does not cross the wire. We need object identity plus principal. +- **Single process with no rollout and `restart-required`** contradicts our goals. That is exactly where their simplicity comes from. + +### Proposal: a constrained model for natives + +1. **Remove `self.resource` from plugin code.** A native exists only inside a supervisor: host values from the base bundle (process, PTY, socket or connection with reconnect, SQLite or plugin database, sandbox, watch). Plugins get handles: opaque ids plus methods plus a status readable. +2. **Every native has an owner** passed at creation: an object identity or a job id. It closes when the owner ends. Keep-alive and presence count natives per owner. +3. **A plugin reload never touches natives.** It reruns factories and restarts routines. Behaviour that used to sit inside resources (terminal logic, search, radius) is plain code around a handle id, so code changes need no version bump. +4. **A plugin that truly needs its own native** (rare) serves it as an object type. A code change restarts that object, and consumers re-hold through references. No `replace`, `deps`, resource versions, ordering rules or stale handles. +5. **Per-host install is all-or-nothing** with rollback; drop "facet keeps its previous version". +6. **Pending waits (Questions, logins) belong to the asking call.** A plugin reload or crash fails the call retryably, and a re-ask with the same `key` reattaches. The questions themselves are documents; resolvers are never kept. + +This deletes most of §12 and part of §15, and reduces the native problem to objects plus references. The cost is a small set of supervisors in the runtime, which has to be designed well because changing it needs a restart. Any plugin native that does not fit a supervisor is restarted on reload, as DSH does. + +## Additional observations and questions from the research commentary + +These record material produced before the final report, including qualifications that the report compressed too aggressively. They are research observations or open design questions, not owner decisions. + +### Lifecycle effects and dependency cascades + +- Cordis is the framework associated with Koishi. Its shared context holds services; services are supplied by plugins. Plugins contribute services, events and reversible registrations. There is no need to import concrete implementations to find services. +- Reloading a plugin means disposing all its fibers and waiting for quiescence, then re-plugging with the same config—essentially a kill-and-restart where natives die. Required `inject` dependencies make consumers pending until providers exist. When a provider disappears or changes fiber uid, its dependents unload and reactivate. +- The epoch is just the concatenation of injected services' provider fiber uids, so when a provider reloads with a new uid, every consumer's epoch shifts and triggers a full unload-and-reload cascade through dependents—no handover or versioning needed, just a clean restart. +- Tools register themselves into `ctx.tools` as registry effects. Reloading a tool plugin removes and re-adds its registration without necessarily restarting the agent loop, since the loop injects the registry, not every individual contributing plugin. Reloading a foundational service such as sessions is a different matter. +- This distinction is important for registry readiness: dependency readiness is not the same thing as an immutable, gapless contribution table. Cordis restart semantics do not solve the pinned Durable phase problem. +- Open investigation question raised during research: does a tool plugin reload cancel an in-flight tool call, or does that call continue on its captured definition while the registry changes? The report inferred continuation for old closures; no universal tool-specific cancellation guarantee was established by the inspected README snippets. + +### Ownership and code lifetime are different + +- The subprocess service keeps process lifetime across consumer reloads, with natives owned by a stable service while consumers hold handles. Reloading the consumer is different from reloading that service. +- The job registry attaches its cleanup effect to the owner's (agent's) context rather than the producer's fiber—effects attach to whichever scope owns the domain, not the scope where the code runs. That's a useful generalization: lifetime belongs to the owner's scope, while the code itself belongs to the producer. +- A running job keeps executing old code from before a producer reload, since its lifetime is tied to the owning agent session or service rather than to the code fiber itself. `jobs-local` explicitly documents this and attaches cleanup via `owner.ctx.effect` (src/index.ts:610-627). +- On terminal backend reload, existing sessions keep running old backend code since they don't do versioned replacement—only a full service reload kills all sessions. That aligns with part of our own approach. `registerBackend` removes only the backend contribution; published records retain a `TerminalBackendSession` object (packages/terminal/terminal/src/index.ts:125-140, :185-197). The terminal-bash README also says the owner policy fence outlives a provider reload that retains existing sessions (:111). +- The terminal service tracks unpublished spawns as reservations, owns an abort signal for them, and waits for their rollback before disposal completes. Partial startup cleanup failures remain owner activity, rather than allowing the owner to look idle while resources may be leaking (terminal README:81-87). +- Jobs teardown cancels work and awaits settlement. A throwing cancel force-fails the record and reports a possible orphan. A cancel that returns without actually settling can stall teardown (packages/jobs/jobs-local/src/index.ts:634-687). Quiescence is a cooperative contract, not an absolute guarantee that arbitrary plugin code can be stopped. +- Job preflight requires a controller serving the owner before starting background work, so the runtime does not launch work the agent cannot later collect or stop (packages/jobs/jobs/README.md:41-43). +- The job registry reports running/stopping jobs as session activity, and session archive requests stop them. This is explicit domain activity accounting rather than generic reference reachability. + +### Processes, sandboxing, IPC and connections + +- Managed subprocess lifetime is stronger than direct child exit: `done` reports the direct command outcome, while `waitForExit()` waits for the provider's managed range to be quiescent. Providers use systemd scopes or Windows Jobs where available and document weaker fallbacks. Consumers own deadlines, termination ladders and cause classification (subprocess README:68-76, :98, :109-113, :151). +- Processes, persistent shells, language servers and external subagent backends share `ctx.subprocess`; shell/tool semantics stay outside that service (packages/subprocess/README.md:12, :35). +- The subprocess control-pipe pattern optionally returns a raw `Duplex` at `handle.control`. The child opens fd 7. Consumers own bounded framing, validation, backpressure and endpoint closure. The provider separately tracks endpoints after managed-range exit and closes leftovers during disposal (.agents/notes/implemented/architecture/2026-09-11-subprocess-control-pipe.md:13-21). +- That control pipe grants no host capability: the child remains untrusted and normal tool dispatch/approval still applies. Model code runtimes such as PTC use out-of-process execution, but this does not mean every plugin native lives behind IPC or in a dedicated supervisor process. +- Filesystem and subprocess providers can be replaced together to move the execution world remotely without changing Bash, PTY or LSP consumers. This is a capability-service abstraction, not Chord-style arbitrary remote objects. +- MCP reconnection is local policy inside a concrete connection supervisor: initial discovery is awaited; lost connections use exponential backoff and an attempt budget; last-known tools remain listed during an outage but calls fail; budget exhaustion removes the tools. Tool generations are refreshed after reconnection. After HTTP is connected, SDK transport recovery handles request failures rather than respawning every connection (mcp-client README:89-93, :127-129, :210). +- This suggested a supervised connection object exposing status rather than raw socket access, not a general runtime ability to preserve arbitrary native values through new code. + +### Host/client and extension model + +- The normal architecture is a Node host plus browser clients, not a p2-style independently reloaded server/session-worker/browser/TUI cluster. Desktop adds an Electron carrier and a private host process; Node IPC carries boot injections, readiness, fatal errors and shutdown, while Web owns RPC and streams (docs/architecture.md:49-53). +- Browser RPC uses generated `@Remote` or `@RemoteScope` methods, declared wire identities, runtime lookups and generated codecs. Complex host objects do not directly cross the wire. Stream remotes use a multiplexed WebSocket or in-process carrier; client-to-host uplink items are validated (docs/api-gateway.md:9-15, :56-60). +- Browser plugins use a separate lazy module system plus Cordis lifecycle. Host bundle changes are observed and sent through SSE. Reload gives a fresh fiber and component state. Bootstrap replacement is refused and requires a page reload; static assembly libraries and shell changes require a rebuilt page. Self-reload closes/reopens SSE and a full snapshot repairs missed graph changes (2026-07-23-client-plugin-loading-model.md:79-97). +- HMR is configuration-only by default in the base bundle (`root: []`). Source-module watching is optional. Headless, SDK and ACP disable HMR by default; without it changes apply on restart (packages/boot/hmr/README.md:27-34). Thus not every DSH profile promises live source reload. +- Installed package replacements require a process restart; source module HMR uses Node loader internals, with explicit limitations for framework dependencies, TSX loaders, ordinary application-created Workers and package-cache refresh (hmr README:92-101). +- Dynamic extensions are process-local definitions with immutable package versions. Host halves run in a `node:vm` realm; browser halves have a separate runner and approval path. Stop disposes effects but retains definitions; undefine removes them. Definitions disappear on restart and page reload requires an explicit run for the client half (packages/extensions/cordis-host-runner/README.md:12, :54-58, :76). +- That VM isolates globals but is explicitly not a security boundary; exposed services reach the live runtime and should be treated like bash access (same README:62). Persistent plugin installation is handled through Plugin Manager instead of a model tool creating transient definitions. +- DSH capability seams separate Service Definition, Service Provider and Consumer roles. The API is pre-stable and all consumers are updated together; this is not p2's requirement that independently built plugins keep working across application versions (AGENTS.md, “Pre-stable APIs”; docs/architecture.md:130-134). + +### State, persistence and user interaction qualifications + +- Session events are durable facts; agent and capability events are live extension points. Model-visible input must be reconstructable from the session log (docs/architecture.md:67-75, :118-126). +- A hard process loss before an assistant stream settles leaves no durable attempt stream. The log is the source of settled history, not a record of every transient byte (architecture.md:120). +- Session persistence acquires explicit write handles. Reloading a backend does not silently re-adopt old handles; old writers fail loudly until sessions restart. This consciously deletes HMR adoption/reseeding machinery (2026-08-27-handle-based-session-persistence.md:9, :27, :35). +- Historical session migration selects the highest committed generation, refuses future versions, migrates through adjacent steps and publishes a verified successor for a write open, preserving source generations. SQLite schema versions are monotonic (architecture.md:122; DSH AGENTS.md). +- Non-session application state is not forced into the session log: storage-json/storage-sqlite plus schema-validated storage-domain records are separate. This broadly matches the proposed split between Chord app documents and Durable transcript-related state. +- Durable scheduling does not make jobs durable. The shipped jobs contract holds callbacks and exact live `Agent` identities, so cross-process or restartable work needs a redesigned seam. +- The general observation “automatic work does not resume by itself” was specifically supported for goal activation; it must not be read as applying to schedules, which intentionally deliver overdue reminders. +- Questions can be reconstructed from tool call/result/session events. While a call is live the answer resolves that call. After timeout/process loss it becomes continued work and a later answer is steered as a new input; it does not fabricate a result for a finished tool call. A disconnected client can reconnect and answer a still-open request (user-questions README:27-40). +- Timed questions separate foreground waiting from question lifetime. A client may claim the wait and own its countdown; if no client claims it, an unattended timer releases the foreground request at the original deadline without aborting the turn. Unfinished draft input remains browser-local (same README:27, :40, :57). +- A human question with an agent authenticates exact live identity and only admits a runtime root, not a child. Agentless local requests may use unscoped listeners and fail without a provider (same README:33). +- Approvals have fresh request ids, a closed allow-once/rejected/cancelled/unavailable outcome and a log-only audit pair. Missing, throwing or nonconforming answerers are unavailable, not permission (docs/subsystems/approval.md:5-20). + +## Open design questions and recommendation details + +These were raised or implied explicitly while forming the proposal; none was settled by the owner in this conversation. + +1. **How narrow can native ownership be?** Dedicated native-owner objects are one option; base-runtime generic supervisors are the stronger restriction. The latter lets plugin code reload while the actual process/socket/PTY remains owned by unchanged code. The former kills/reopens the native when its provider reloads. +2. **What counts as data versus behaviour?** A plugin class instance holding methods/listeners is code, not a passive native container. Rebuild it on reload and retain its state in documents. Saying “behaviour belongs in routines by convention” does not enforce this distinction. +3. **What is the minimum supervisor set?** Proposed primitives were process, PTY, socket/connection with reconnection, SQLite/plugin database, sandbox and filesystem watch. A small rarely changing kernel avoids moving every plugin into the base bundle, but the available primitives and their interfaces still need design. +4. **What must a native survive?** Tool/consumer reload, owning-object release, native-owner code change, worker idle stop, server restart and network reconnect are different events. DSH distinguishes some of these through service ownership but does not supply our general live handover or cross-host rollout guarantee. +5. **How should unsupported custom natives behave?** Proposed escape hatch: a plugin serves a native-owning object and that object restarts on code change; consumers get an error and re-hold. This gives up continuity for that native and conflicts with parts of the current resource target. Owner decision required. +6. **Can references replace resource `deps`?** Restarting a provider and re-holding through normal object references would remove resource topological replacement, handover and dependent reopening. This must not be confused with an already implemented transparent restart cascade across remote hosts. +7. **How much all-or-nothing publication is possible?** The proposal was to refuse the whole host install when a facet setup fails, rather than preserve an old facet inside a newer host revision. Async native activation, migrations and effects make stronger rollback difficult. DSH restores modules/plugin implementations, not external world state. The recommendation needs a precise boundary rather than claiming a transaction over native effects. +8. **How are Durable phases pinned?** Keep immutable registry tables and allow old calls to retain their old definitions. DSH's fresh-fiber cascade and deletion/re-registration gap do not meet p2's pinned-phase and readiness requirements. Moving natives behind stable methods removes raw-resource captures, but does not by itself settle which version an old phase should call. +9. **How should Questions survive reload?** One possibility was document-backed questions plus call-scoped resolvers: an interrupted ask fails retryably, and re-asking with a stable key reattaches. DSH's actual post-loss model instead sends a new user message. These are alternatives, not the same semantics, and neither has owner approval here. Logins, approvals and quorum/all-audience questions need their own answer-loss policy. +10. **Who computes keep-alive?** Proposed answer: the host counts references, jobs and supervised natives by owner so plugin authors do not maintain independent liveness flags. A native is activity, but not necessarily a viewer; counting every native as presence would conflate those concepts. The target already distinguishes lingering keep-alive from actual presence. +11. **How does ownership cross hosts/principals?** Exact `Agent` object identity is trusted same-process identity, not a portable principal or capability. p2 needs runtime-stamped object/job ownership and authority checks on methods reached across links. An opaque id alone does not authorize access. +12. **Which simplifications are unacceptable?** Killing agents on reload, package replacement requiring application restart, resetting component state, single-host assumptions, updating every API consumer in lockstep and in-process job callbacks are all parts of DSH's smaller problem. They cannot be adopted wholesale while keeping p2's stated requirements. + +## Qualifications to the original report's broad claims + +The original wording is preserved above for completeness, but it should not be treated as a proof of stronger guarantees than the inspected evidence supports: + +- **“Natives are never values in plugin code” / “plugins hold ids” is a proposed generalization, not a universal DSH rule.** DSH passes live handles, backend session objects and job callbacks in process. The subprocess control channel explicitly returns a raw `Duplex`. Dedicated services are a useful ownership pattern, not a ban on all raw native access. +- **“No native survives a code change of its owner” depends on which owner is meant.** A backend plugin can unload while the stable terminal service retains sessions containing old backend behaviour. Job closures survive producer reload. The lifetime owner and implementation author are not necessarily the same plugin. +- **“No in-memory state survives a reload” only applies to the replaced fiber and rebuilt UI state.** Other services, retained sessions and running job closures can preserve memory and old code. This is precisely why lifetime ownership must be distinguished from code generation. +- **“No mixed versions” describes the intended host source-HMR replacement/rollback path, not a cluster-wide guarantee or an absence of old closures.** Browser reload is separate; failures can remain visible and retryable without rolling back unrelated browser plugins. Host rollback reactivates implementations but cannot undo arbitrary external effects or recreate killed PTYs. +- **“Effects unwind in reverse order” needs scope.** The fiber's documented effect API promises reverse order within an effect's collected disposers (vendor/cordis/src/fiber.ts:404 onward), but `_unload()` awaits cleared top-level disposers using `Promise.all` (:675-694). Ordered teardown requires bundling related work into one effect, as docs/cordis-primer.md:45 explicitly advises. Do not infer a universal dependency-safe sequential shutdown from the short report. +- **“One host” is the ordinary product composition, not an absence of child processes, remote execution or the desktop host/client split.** The useful contrast is absence of Chord-style many-host object rollout and principal-based binding, not absence of IPC. +- **“The session log is the only durable truth” is about session/model history.** Non-session application records and schedules are independently durable, as the report also notes. +- **“Gives up nothing” in the comparison table is shorthand for the target's broad ambitions, not a claim that Chord has no costs or constraints.** Chord already forbids native sharing across plugins, constrains transactions and restricts reload publication. The proposal is about further narrowing what plugin authors may retain. +- **“No pending promise needs to survive anything” means DSH does not promise resuming a lost JS continuation.** Live question services still hold pending waits and timers; durable projections preserve question facts, not promises. +- **“Killing live agents conflicts with live reload” is a continuity trade-off, not a logical impossibility.** It conflicts with the supplied stronger requirement that running work finish on its old code without reload draining it. +- Source references were gathered during a read-only research pass. The full target design was read; plugin-loading and Questions were consulted selectively. Not every DSH package or cited implementation was read in full. The archive records the research as produced rather than upgrading its coverage claims. diff --git a/apps/p2/docs/p3-reviews/durable-deferred-commits.md b/apps/p2/docs/p3-reviews/durable-deferred-commits.md new file mode 100644 index 000000000..bfe91f998 --- /dev/null +++ b/apps/p2/docs/p3-reviews/durable-deferred-commits.md @@ -0,0 +1,152 @@ +# Durable: deferred storage of throttled commits + +Status: **proposal for review** (2026-10-24). Replaces the `pi.run`/`pi.progress` split +(`durable-run-progress.md`, kept for its reviews). A pi change in `packages/durable`, upstream first. No schema change, +no change for renderers. + +## Problem + +Durable commits a streaming answer's partial (`pi.live.generation.message`) and running tool output (`pi.live` tool +slots' `output`, `details`, `diagnostics`) about every 100 ms per conversation. Each commit is a full storage +transaction: SQLite writes ~17.7 KB per commit (page granularity) for a ~150-byte change, and p3's store adds an +outbox record per commit. With 32 streams (a real case): ~270-320 commits/s, ~4.7 MB/s of disk writes (~17 GB/hour), +~11% CPU on slayer (Fable, `/tmp/group-commit-fable.md`; Astra, `/tmp/drp-review-astra.md`). CPU is mostly the +Session's in-memory work, which stays; disk and outbox are what this proposal reduces. Group commit was measured and +rejected (2-4x less disk only at >= 8 streams, no CPU gain, a few ms latency on every commit, early adoption needed). + +## Idea + +Only the two throttles (the generation's partial throttle in `harness/generation.ts`, the tool output throttle in +`harness/tool.ts`) mark their commits **deferrable**. Such a commit is adopted in memory and published to watchers +at once, as today, but not stored: its ops wait, folded, in a `PendingOps` per document (Chord's +`@earendil-works/chord/delta`, upstream `d8d99950a`). They are stored by the first of: + +1. the next **normal** commit that writes the **same document** (it stores the held ops first, in its own storage + transaction); +2. a Session **timer**, at most ~1 s after the first held write (one storage transaction for every held document); +3. `close()`. + +A deferrable commit that writes anything else (an entry, a task, another document, a document creation, migration or +retirement) is stored at once: the mark is a permission, not an order. + +## Why storage stays consistent + +- Held ops belong to one document, `pi.live`, and only to its streaming fields. +- Everything that consumes or clears streamed content writes `pi.live` in the same commit: converting a partial into an + aborted entry deletes `generation`; ending an answer clears the partial and sets `tools`; settling a tool builds its + result from the slot's output and sets the slot `done`; steering, run start and run end change `run`. So each of + these commits stores the held ops first, together with its own changes. +- Commits that do not write `pi.live` (a task's own state, a display-only entry, a submission record, another + document) do not depend on the partial; they need not flush it. +- Guarantee: every document in storage is a state it really had; every commit that writes a held document stores it + together with its own changes. Interleaving is low: control changes of `pi.live` happen at phase boundaries (run + start, answer end, tool start and end, run end); streaming happens in between. +- Order: held ops are older than the normal commit that stores them, so they come first in the stored delta; with + folding, a later write of the same field absorbs the held one (deleting `generation` drops the held appends). + +## Adoption, sequence numbers, publication + +All on the Session's line (one commit at a time), as today. + +1. **A held commit:** memory adoption only (the tracker's pointer swap and ops), no change to stored-revision counters + (`storedVersion`, `deltasSinceBase`) or record metadata; its ops go into the document's `PendingOps`; the document + is pinned in the Session's cache until stored (no idle eviction, also with retention 0). Publication is tagged: + + ```ts + type CommitPublication = + | { readonly kind: "stored"; readonly seq: Seq; readonly changes: readonly CommitChange[] } // as today + | { readonly kind: "memory"; readonly after: Seq; readonly changes: readonly DocumentCommitChange[] }; // held + ``` + + A held commit only ever contains document changes, hence `DocumentCommitChange`; `after` is the last stored seq. + The view mount and document watches apply its ops as usual; listeners that care about durability (replication, + anything keyed by seq) skip `memory`. +2. **A normal commit writing a document with held ops:** `take()` the held ops, fold this commit's ops for that + document onto them, one `document.change` write; storage commits as today; `adopt(seq)` adopts this commit's own + ops in memory and updates the stored counters for the whole delta; the publication carries only this commit's own + ops (the held ones were published already); the pin is released. +3. **The timer or `close()`:** a store-only commit on the line, one `document.change` per held document, one storage + transaction; adoption updates stored counters only (no memory change, no publication). It takes a seq like any + storage write; listeners see a gap, which Durable allows (it compares seqs, never counts them). + +So `Transaction.adopt` splits into adopting in memory and recording what was stored; today's path runs both, a held +commit only the first, a flush only the second. + +## Restrictions and edges + +- The option applies only to documents with `history: "latest"` and `fork: "initial"` (`pi.live` qualifies): "as of" + reads and forks of the current state read storage by seq and would miss held ops. +- Creation, migration and retirement of a document are never held. +- Checkpoints: `checkpointWhen` counts stored deltas; a base write includes everything held. +- The timer is a foreground timer (Workers refuse background timers), its deadline from the first held write; a failed + flush fails the Session (as any storage failure). +- Other readers of storage (p3's outbox and replicas, an export, a second process) lag up to ~1 s for held fields; an + export that must include them flushes first. + +## Crash and resume + +After a crash or a Cloudflare eviction, held ops are lost with the process; storage has `pi.live` with control as of its +last normal commit and streaming fields as of the last flush (up to ~1 s older, or absent if the stream started less +than a second before). + +| What was running | Resume | Difference from today | +| --- | --- | --- | +| generation streaming | converts the stored partial into an aborted entry, clears it (normal commit), resends | the aborted entry up to ~1 s shorter; none if the stream was under 1 s old (today: 100 ms); its usage up to ~1 s low | +| deferred response being polled | continues from the task's checkpoint | none | +| replay-safe tool | reruns, progress cleared by a normal commit | none | +| tool without replay | ends `interrupted` with the stored output | output up to ~1 s shorter (accepted lossy recovery input) | +| compaction | reruns from its checkpoint | none | +| clean `close()` | stores everything held first | none | +| restart counter (#11139) | partial commits never counted as progress | none | + +At open nothing is held; every resume step that consumes progress is a normal commit writing `pi.live`, so a second +crash cannot convert the same partial twice. + +## Expected effect + +32 streams: storage transactions for partials and tool output drop from ~300/s to at most one per conversation per +phase boundary or per second; disk and outbox accordingly (estimate: about 10x less; to be measured). CPU about the +same (the 100 ms in-memory work stays). One stream: ~10x fewer storage transactions for streaming. + +## Changelog + +Durable's `CHANGELOG.md`: the tagged `CommitPublication` is a breaking change for `subscribeCommits` consumers (a +migration note: skip or handle `kind: "memory"`); the crash window for streamed content grows from ~100 ms to ~1 s. + +## What the reviewers check (against Durable's code, in full) + +1. The consistency argument: is there any commit that stores data derived from `pi.live`'s streaming fields without + writing `pi.live` in the same commit? Any reader of storage (scheduler, resume, inspect, forks, copies, events, + view builds after eviction) that would see an inconsistent combination? +2. The adoption split, the tagged publication, the flush path, pinning, the timer, `close`, failures: feasible as + described in `session.ts`, `transaction.ts`, the documents and storages? Every consumer of `CommitPublication` and + `seq`. +3. Resume: every path in the table, plus double crashes. +4. Interleaving: measure on a real Durable conversation with tools (parallel tools included) how many normal commits + write `pi.live` per second against held ones; and the disk, outbox and CPU effect at 1 and 32 streams. +5. Size of the change and anything simpler. + +## Decided (owner, 2026-10-24), after the reviews (`/tmp/ddc-review-{astra,fable,opus}.md`) + +- **Spec change accepted:** Durable's spec §1 ("all visible progress is durable; no volatile publication path") and the + §13 non-goal change: a commit may be marked deferrable; what watchers see can be up to its window ahead of storage. + A stream that crashes before its first flush leaves no aborted entry, and that attempt's usage is not recorded. +- **Public option, per commit, tagged:** `commit(change, context, { persistence: { kind: "deferred", withinMs? } })`; + without `withinMs` the default is `settings.progress.persistWithinMs` (1,000); without the option, immediate. No + internal flag: any task (built-in or third-party) may use it. The two throttles (generation partials, tool output) use + it. +- **A deferrable commit is held only** when every write changes an existing document defined with `history: "latest"` + and `fork: "initial"`; creation, migration and retirement of a document are never held; anything else in the commit + makes it immediate (the mark is a permission). +- **Stored by the first of:** a normal commit that **reads (`tx.doc`) or writes** the document (held ops first, in its + storage transaction); the document's deadline (the earliest `withinMs` deadline of its held commits), on a + **foreground** timer through the injected `setTimer`; `close()` (after `writeAtClose`, before documents are cleared, + in `try/finally`); and before a tracker with held ops is dropped (`unloadDocuments`, a version-mismatch reload) or + its document retired. +- **Held ops leave the queue only at the last step before `Storage.commit`** (a failure earlier loses nothing); a failed + flush fails the Session; the `PendingOps` overflow (10,000 ops) stores a base. +- **Pinning:** a document with held ops is not evicted. +- **Publications:** held commits are delivered to internal observers (`observeCommits`: the view mount, document + watches; tasks and submissions ignore them) tagged memory-only; `subscribeCommits` stays durable-only and carries + exactly what was stored (no breaking type change for it). +- **Changelog:** a Breaking/Changed entry for the spec change with a migration note (opt-in; what a crash loses). diff --git a/apps/p2/docs/p3-reviews/durable-run-progress.md b/apps/p2/docs/p3-reviews/durable-run-progress.md new file mode 100644 index 000000000..f3b8800a6 --- /dev/null +++ b/apps/p2/docs/p3-reviews/durable-run-progress.md @@ -0,0 +1,124 @@ +# Durable: split `pi.live` into `pi.run` and `pi.progress`, with deferred persistence for progress + +Status: **proposal, owner wants it (2026-10-23), pending a full review by Astra and Fable**. A breaking change in pi's +Durable (`packages/durable`), upstream first; pairs with pi #11327 (windowed view, `runFailure`, removing +`watchEvents`) and #11139 (a run cut off by repeated crashes ends visibly). + +## Problem + +Durable commits a streaming answer's partial every ~100 ms per conversation into `pi.live`, a persisted document. Each +partial commit is a full durable write: a storage transaction, a delta row, an outbox record (p3's store syncs it to +Radius). With 32 subagents streaming in one session (a real case) that is ~320 durable commits per second, almost all +overwritten 100 ms later. Measured by `rqlite-eval` (`/tmp/rqlite-eval.md`, real Durable Session on SQLite, 32 +streams): 100 ms persistence 9.8% CPU, 5.6 MB/s disk (20 GB/hour), 79 KB/s outbox raw; 1 s persistence 1.6% CPU, +0.6 MB/s, 16 KB/s. Per partial commit ~640 us, ~310 us of it the SQLite transaction (`is-9265`'s profile). + +Persisting the partial less often is not possible today, because `pi.live` mixes two kinds of fields: control that +Durable itself decides with (`run`, tool slot identity and status, compaction status) and fields only clients display +(the partial message, tool output, backoff times). Deferring writes of a mixed document either risks losing control +state in a crash (trace: the final commit deletes `run` and settles the submissions; if `pi.live`'s part were lost, the +resumed host sees `run` still set, `submissions.ts` treats the conversation as busy, nothing ends that run) or needs a +flush rule that, applied session-wide, would flush ~10 times per second with 32 streams and win nothing. + +## Proposal + +### Two documents, every field in exactly one + +```ts +/** pi.run: what Durable decides with. Persisted with every commit, as today. */ +type RunState = { + run?: { taskId: TaskId; inputs: SubmissionId[] }; // present exactly while busy + runFailure?: { taskId: TaskId; reason: string; error: string }; // why the last run ended unanswered (#11327, #11139) + generation?: { attempt: number }; // an attempt is running + tools?: { callId: string; name: string; status: "pending" | "running" | "done"; taskId?: TaskId; entry?: EntryId }[]; + nestedTools?: { + callId: string; name: string; status: "pending" | "running" | "done"; + parentCallId: string; parentTaskId: TaskId; taskId: TaskId; + arguments: JsonObject; // what the call runs with + summary?: { isError: boolean; durationMs?: number; usage?: Usage; error?: string }; + }[]; + compactions?: { taskId: TaskId; reason: CompactionReason; blocking: boolean; attempt: number }[]; +}; + +/** pi.progress: for display only. Persisted deferred (at most once a second). */ +type ProgressState = { + message?: AssistantMessage; // the partial answer + retry?: { at: number; error: string }; // generation backoff + deferred?: { pollAt: number }; // a deferred response being polled + calls: { [taskId: string]: { // model-issued and nested calls, by tool task id + output?: string; droppedBytes?: number; droppedLines?: number; + details?: JsonValue; diagnostics?: ToolDiagnostic[]; + } }; + compactions: { [taskId: string]: { retry?: { at: number; error: string } } }; + compactionFailure?: { taskId: TaskId; reason: CompactionReason; error: string }; +}; +``` + +- Progress is keyed by **task id**, not call id: a nested call's id is `<parentCallId>/<key>` and a model-issued id + from OpenAI Responses (`call_id|item_id`) may contain `/`, so call ids are not unique across both kinds by + construction; task ids are (minted per session), and progress only exists while a call runs, when it has its task. +- Renderers read both from one view value and join by task id; nothing is duplicated, so nothing can disagree. +- `pi.live` disappears (the name meant "live state"; `pi.run` is control, `pi.progress` display). + +### Deferred persistence: a generic document option + +```ts +export const ProgressDoc = defineDoc<ProgressState>({ kind: "pi.progress", …, persist: { deferMs: 1000 } }); +``` + +The Session knows nothing about `pi.progress`; its rule is generic: + +- Every commit is adopted and published at once, as today (watchers and the view see every 100 ms update). +- Writes to documents whose definition has `persist.deferMs` are **held** per document, their ops folded (the leaf + index of Chord's pending queue: appends to one text join, sets replace sets); all other writes go to storage in the + commit's storage transaction as today. +- One Session timer stores every held document's folded ops in one storage transaction, at most `deferMs` after the + first held write. A commit that writes a held document anyway may carry its held ops in the same transaction + (optional; storage is fresher at no extra transaction). +- `close()` stores everything held. A base write of a held document (checkpoint, retirement) includes its held ops. +- No commit ever has to store another document's held writes for consistency: held writes touch only their own + document, so storing them later gives the same stored state. + +### Crash and resume + +- Control (`pi.run`, entries, tasks, submissions) is always stored with its commit. +- `pi.progress` in storage is up to ~1 s behind, or absent if the host died within the first second of a stream. +- Resume converts the stored partial (`pi.progress.message`) into an aborted entry as today; if it is absent, there + is no aborted entry and the request is resent anyway. + +### Migration of existing sessions + +- `pi.run` is created from an existing `pi.live` (its control fields) when a conversation is first opened by the new + version; `pi.live` is retired. +- Every view-only field is cleared when a run ends (`endRun` deletes `generation`, `tools`, `nestedTools`), so a + session at rest loses nothing; `pi.progress` starts empty and is created by its first write. +- A session upgraded in the middle of a run (it crashed or stopped mid-stream and resumes on the new version) loses its + old partial preview and tool output; resume may read the old `generation.message` once to write the aborted entry. +- Clients reading `pi.live` break (public protocol, spec §9.3): p2's renderers, aron-cf's integration. Accepted + (owner). + +### Changelog: breaking change with a migration guide (owner) + +Durable's `CHANGELOG.md` gets an entry under `### Breaking Changes` with a migration guide inside it: what moved where +(a field table `pi.live.<field>` -> `pi.run.<field>` or `pi.progress.<field>`), how a renderer joins tool slots with +their progress by task id (code before and after), what the view exposes (`docs["pi.run"]`, `docs["pi.progress"]`), +what happens to stored sessions on first open (and mid-run), the new `persist: { deferMs }` option for extensions' +own documents, and what a crash now loses (up to ~1 s of progress, or the whole progress of a stream's first second). + +## What the reviewers check (in full, against Durable's code) + +1. Every reader and writer of each `pi.live` field (`generation.ts`, `tool.ts`, `compaction.ts`, `live.ts`, + `submissions.ts`, `events.ts`, `view.ts`, `inspect()`, the task graph, tests): is each field placed right (control vs + display)? In particular: does Durable read `generation.retry`, `generation.deferred`, a compaction's `retry` or a + tool slot's `output`/`details` itself (resume, scheduling, replay)? Then it is control and stays in `pi.run`. +2. The Session mechanics: adoption and publication of a held write without a storage seq (today adoption is driven by + storage's `adopt(seq)` callback and every publication carries a seq); the documents engine accepting one stored + delta that covers several in-memory revisions; checkpoints (`checkpointWhen`), the "complete base whenever nothing + runs" rule of `pi.live`; retirement; `close`, eviction and `writeAtClose`; `StoreStorage` and the outbox (p3's store). +3. Crash and resume: every resume path that reads progress (`convertPartial`, tool replay clearing progress, compaction + retries), and what an absent `pi.progress` means for each. +4. Migration: creating `pi.run` from `pi.live` on open (where, when, atomically), mid-run upgrades, forks + (`fork: "initial"` today), copies (`document.copy`). +5. The view (#11327 and today's mount: `MOUNTED` documents), `watchEvents` until it is removed, and p2's renderers. +6. Numbers: confirm the win at 32 streams with the real code path (commits, CPU, disk, outbox), and what remains. +7. Anything simpler that gets the same result. diff --git a/apps/p2/docs/p3-reviews/pl-agent.md b/apps/p2/docs/p3-reviews/pl-agent.md new file mode 100644 index 000000000..b39a936d8 --- /dev/null +++ b/apps/p2/docs/p3-reviews/pl-agent.md @@ -0,0 +1,531 @@ +# pl-agent — models, prompt, code mode, agent, logs, developer tools, PowerShell, web and TUI design review + +Status labels in this document: + +- **DECIDED**: explicitly decided by the owner in `redesign-decisions.md` or its porting guidelines. +- **RESOLVED / SUPERSEDED**: the later p3/Chord v2 design removes the reviewed mechanism or adopts the recommendation. +- **OPEN**: explicitly listed as open in `redesign-decisions.md`, or still not answered by the decisions known to this reviewer. +- **RECOMMENDATION**: proposed by this review, not known to be accepted. + +This records the design findings and recommendations from two rounds: the review of `target-design.md` at `6559589bd`, then the ownership/native-object review after `redesign-decisions.md` at `5abe3f6d6`. It intentionally excludes the later functionality map. + +## Round 1: adversarial review of the original target design + +The reviewed plugins were models, prompt, codemode, coding-tools, agent, web-shell, logs, dev-tools and powershell, including their interactions with login, sessions, chat, subagents, environments, Durable, web and TUI runtime services. + +### Summary verdict and cost + +**SUPERSEDED estimate, still useful as scope evidence.** The initial estimate was about 16 files and 650–850 changed lines. models was the largest part (about 300 lines: `resources.ts` restructured, methods replacing the opaque member, derives). Then prompt (about 150), logs (about 80), codemode (about 50), powershell (about 40), dev-tools (about 30), and agent, coding-tools, web-shell and the TUI/web registrations (about 10–20 each). Login and sessions added about 100 lines for removing models' opaque native, and subagents about 40 lines for removing the shared document write. + +The original verdict was: + +> **The design can express these plugins, with conditions.** Findings 1, 4 and 6 are API gaps (logic refresh of kept natives, overlap policy, rebinding derives over native change sources, whole-value replacement); without them, reloads of models, prompt and logs succeed without changing behaviour. Findings 2, 3, 5 and 7 are lifecycle and cross-plugin cases that need small additions: a close reason, deferred client releases, rebuilding dependent facets on reload, and handing over the credential locks. + +The later p3 design deliberately replaces rather than migrates p2, and replaces resources with fixed-lifetime native objects, so the implementation estimate and several mechanisms are superseded. The concrete traces remain relevant where noted below. + +### 1. Kept resources silently keep old plugin logic + +**Status: RESOLVED by the decision that native objects restart on plugin reload; the underlying restart cost moved to later findings.** + +> A kept resource keeps running the code that opened it, so a reload reports `installed` and changes nothing. + +Concrete traces: + +- models: `Resources` (`models/resources.ts:41`) wires its watcher to its own class method `load` (`resources.ts:67-79`). `load` calls `applyModelConfig`, which builds providers and patches `models.getAuth` with closures from that code (`provider-config.ts:673-691`). `FileCredentials` and `FileModelsStore` are plugin classes too. +- Trace: fix a bug in `provider-config.ts`, then reload models. The resource keeps its version, so it is kept. The next `models.json` change runs the old `load` with the old `applyModelConfig`. The fix only lands after a restart or a version bump. +- prompt: the same problem exists through closures in `prompt/worker.ts:84-106`, `environment.ts:89-97` and `context-files.ts:64`. A fix to skill discovery never applies; even a session move reuses the old closure (`worker.ts:154-158`). +- logs: the `setInterval(collect)` closure in `logs/server.ts:21-67` keeps running the first version of `collect`. +- Each such resource pins its first code generation, working against loader D's generation-GC goal. +- The old rule “keep logic out of the resource” could not actually be met by models: pi-ai's `MutableModels` is a native whose content is plugin closures. + +Original recommendation: + +> Add a resource mode `version: "code"`: every reload of the plugin runs the new code's `replace(old)`, which keeps the natives (file locks, watcher) and rebuilds the logic parts. Add a static check that warns when a resource's `open` returns an instance of a class defined in the plugin. + +That recommendation is superseded. The owner instead decided that natives are fixed for one object lifetime and native objects restart on reload. This guarantees new code but makes expensive or disruptive restart behavior explicit. + +### 2. Replacing or restarting the code-mode sandbox can hang Durable work + +**Status: OPEN. `redesign-decisions.md` explicitly lists close-with-reason as raised and not yet discussed. The new design makes the trigger more common: every reload of the native-owning plugin restarts it.** + +> Replacing the codemode sandbox hangs running scripts instead of failing them. + +`codemode/calls.ts:17-23` records the reason: when the sandbox ends a run on its own, Durable stores no outcome, and the call waits forever. + +Original trace under the resource design: + +1. Bump the `sandbox` resource version (`codemode/worker.ts:22`). +2. Resource replacement closes the old sandbox. +3. `calls` depends on harness and worker, not on the sandbox (`worker.ts:27-34`), so `RunningCalls.cancel` does not run. +4. The tool call never settles, the conversation remains busy, and the worker never goes idle. + +The p3 trace is now: + +1. Reload the plugin while a code-mode script is running. +2. The native object is restarted and closes `CodemodeSandbox`. +3. Durable sees that the sandbox abort was not initiated through the invocation's own controller; its current behavior stores nothing so a future worker can replay it. +4. The current invocation can remain waiting forever unless close explicitly settles or aborts the Durable task. + +Related problem: + +- On a worker stop, code mode must **not** abort and commit a terminal outcome, because the next worker should replay the interrupted script. +- On a plugin reload/removal, the running call should be deliberately aborted and settled. +- Today this distinction is inferred through `worker.liveness.stopping` (`codemode/worker.ts:31-33`). + +Original recommendations: + +- Under the resource design, make `calls` depend on `sandbox`, so calls close and settle before the sandbox closes. +- Pass `close(value, { reason: "replace" | "remove" | "stop" })`. +- Replace the raw `setTimeout` poll in `reported` (`calls.ts:67`) with a Durable watch. + +Current recommendation: + +- Chord v2 native close needs a reason, at least `reload`, `end` and `stop`. +- The code-mode native owner must abort the Durable tasks it serves on reload/end, but not on worker stop. +- Define how a native owner accounts for and settles non-Chord work, because “running work in native objects is aborted” currently describes Chord work, not Durable task invocations hidden behind a plugin extension. + +### 3. Client renderer reloads create false departures + +**Status: OPEN. `redesign-decisions.md` explicitly lists “UI renderer reload vs. leaving” and presence behavior as open.** + +> A web or TUI facet reload releases every reference below it, and the design reads a release as someone leaving. + +Concrete traces: + +- web-shell registers the root component (`web-shell/web.tsx:46`). Giving `App` a new component identity remounts the whole tree. Every `useObject` cleanup calls `ref.release()` (`runtime/web.tsx:177-181`). +- Login: `ProviderLogin`'s viewing cleanup runs `abandon` (`login/server.ts:134-141`), so reloading the shell can kill an OAuth login in progress. +- Questions: with `all` and a live audience, a reloading owner appears to leave. If everyone else has answered, the question can resolve with that owner in `missing` even though the owner did not intentionally depart. +- Presence: viewer counts briefly fall and rise. +- TUI: when a slot factory changes, the runtime disposes its surface (`runtime/tui.ts:473-479`). `ModelState` releases and re-holds `Chat` (`models/tui.ts:73`). +- TUI notices: `NoticeWatch` resets `seen` and `first` (`logs/tui.ts:164-165`), so old notices are summarized again after reload. + +Recommendation: + +> While a client install runs, defer releases until after the next render commit, and let a re-hold of the same `(type, input)` take over the old remote hold. Neither `host.references` nor the linger bit should see this as a departure. + +This also needs a decision for full web reloads/TUI worker swaps, where the old and new clients are different runtime instances but represent one logical viewer origin. + +### 4. `routines.on` needs explicit overlap semantics + +**Status: OPEN. `redesign-decisions.md` explicitly lists `routines overlap: "latest"` as open.** + +> `routines.on` does not say what happens when a new value arrives while a handler runs; PowerShell and prompt need the old run aborted. + +Concrete traces: + +- PowerShell probes an unreachable remote in a loop until its own “moved” signal aborts (`powershell/worker.ts:32-37,44-53`). +- Prompt closes the old watcher after a location move (`prompt/worker.ts:154-160`). +- A session is on unreachable remote A, then moves to a local Windows machine. If `on` queues handlers, the new handler waits behind A's endless retry, so PowerShell remains stale and is not offered. +- On reload, serialized routine restart can wait behind the same handler until its deadline. +- If prompt-watch logic moves into a restarted routine, its “first load done” marker cannot live in that routine. Otherwise every prompt-plugin reload makes subsequent input wait for another remote scan, up to the existing 60-second load timeout. + +Recommendation: + +> `self.routines.on(source, fn, { overlap: "latest" | "queue" })`. `latest` aborts the running handler's `ctx.signal` when a new value arrives. Durable “first load done” markers belong in state that survives routine restart. + +### 5. Agent copies and writes another plugin's Durable document + +**Status: RESOLVED / DECIDED in the p3 porting guideline. Core-enforced conversation policy is written at child creation; plugins do not share private documents.** + +> agent copies another plugin's Durable document and writes to it. + +Concrete trace: + +- `agent/tool.ts:17` imports `Subagents` and `resolveChoice` from `subagents/tools.ts`, a loader-D cross-plugin copied module. +- It writes `registry.oneShots` into the subagents-owned document (`agent/tool.ts:82-90`). +- If subagents bumps `Subagents` to version 2 and only subagents reloads, agent's bundle still has the version-1 copy; `tools.agent` throws on the newer document. +- Reverse case: agent reloads with a version-2 copy while the subagents worker facet rejects and remains at version 1. agent writes a v2 document; subagents' old code then throws on its own document, including `subagentCwdFollowers`, which relocation/start calls (`runtime/worker.ts:476-526`). The relocation fails and can leave recovery to a worker restart. + +Original recommendations: + +- Expose a host-local recorder owned by subagents, or add a Durable commit hook, so agent no longer imports/writes the document. +- Rebuild every facet whose bundle metadata includes a changed cross-plugin file, even when only one plugin is named. +- More generally, forbid copied cross-plugin modules from defining Durable documents or tasks unless all consumers form one reload unit. + +Owner decision: + +- Core owns per-conversation policy (`readOnly`, `followsSessionCwd`, and equivalent creation metadata). +- Agent/subagents write that core format in the creating commit. +- Chat and relocation enforce it directly. +- `ReadOnlyConversations` and `SessionCwdFollowers` policy registries go away. + +The one-shot listing metadata still needs one owner; it should not return to a plugin-shared private document. + +### 6. Old derives over changing natives and whole-root document writes + +**Status: native-source half RESOLVED by fixed-lifetime natives and native-object restart. Whole-root replacement remains UNKNOWN/OPEN unless Chord v2's document API has since specified `set`.** + +Original finding: + +> Derives over a changing native do not work as specified, and documents cannot replace their root value. + +Concrete traces: + +- models' `summary` and `snapshot` follow changes inside `Resources` (`models/server.ts:31-39`, `config.ts:51-59`). A derive over the resource handle only recomputes when the handle rebinds, not when the native changes internally. +- Taking a readable out of `.value` was the old resource footgun; `questions.md` nevertheless illustrated `self.derive([pending.value.readable], …)`. After replacement, that derive could stay attached to old data silently. +- PowerShell's `windows` and dev-tools' `offered` replace scalar roots. The document draft design described property assignment, `push` and `Object.assign`, but not replacement of a boolean root. +- `Object.assign` is also not enough for replacement semantics because it does not delete keys absent from the next object. + +Original recommendations: + +- Derive sources shaped as `(handle, value => value.changes)`, rebound as part of replacement. +- Add `handle.set(ctx, value)` and `tx.set(handle, value)` for whole-value replacement. + +The new native design removes replacement/rebinding of a native during one object lifetime. Native owners still need explicit source readables for internal changes. The scalar/root-document write question remains. + +### 7. Models exposes opaque natives across plugin boundaries + +**Status: DECIDED. No natives across plugins; consumers use owner methods/state. Restart behavior during login remains OPEN as a consequence.** + +Original finding: + +> Natives cross from models to login and sessions; the design forbids that, and the replacement methods fail mid-login when models reloads. + +Concrete traces: + +- login constructs `Logins` with `ModelConfig.resources` (`login/server.ts:62`) and calls `models.login`, `refreshModels`, `modelStore.delete`, `reload` and `publish` (`login/logins.ts:41,130-146,199-206`). +- sessions reaches through the same opaque resource to validate a model (`sessions/server.ts:223`). +- `HarnessConfig` similarly returns a native `Models` object to core Harness, chat, agent and subagents. +- Under the old replacement design, an OAuth flow waiting for the browser could lose its credential write when models' old `Resources.close()` closed `FileCredentials` and its lock (`resources.ts:287-295`). + +Original recommendation: + +- Replace the opaque member with methods: `hasModel`, `setDefault`, `setEnabledModels`, login/logout/refresh and related operations. +- Use callbacks only for local methods such as provider prompts/notices. +- Hand locked files over during replacement, and resolve the current credential store at write time. +- Clarify whether host-owned registries such as `HarnessConfig` are an allowed native bridge; otherwise move model runtime into core. + +Owner decisions now known: + +- No natives across plugins. +- Local methods may take callbacks. +- Methods return data, and objects cross boundaries only by address. +- Durable keeps its own types; p3 hands Durable plain data/Durable types, never Chord v2 objects. + +Open consequence: + +- If the native owner includes a provider login flow, reloading that plugin aborts it. The design says expensive natives should be split into another plugin, but the correct ownership/split for models, credentials and provider login still needs to be chosen. + +### 8. Logs duplicate work and scale poorly + +**Status: OPEN.** + +Original finding: + +> Smaller issues in logs: duplicate lines after a reload, and per-line cost. + +Concrete traces: + +- If detached-log collector dedupe state (`seen`, `fingerprints`) lives in restarted behavior (`logs/server.ts:27-28`), reload can reappend every line from `terminals.logs.json` and `dev.logs.json`. +- `Logs.lines` copies and diffs up to 1,000 lines for every appended line, per open view (`runtime/logs.ts:177`, `logs/server.ts:96-99`). A burst with several open views amplifies copying and diff cost. + +Recommendations: + +- Keep detached snapshot dedupe state with the owning process-log store rather than with the restarted collector behavior. +- Prefer a store-backed readable that publishes append and front-trim operations instead of copying/diffing the full 1,000-line array for every line. +- Longer term, push detached process logs over IPC rather than polling snapshot files, while retaining an exit snapshot if startup/fatal errors require it. + +### Additional round-1 review points + +#### Immutable Durable registry snapshots are the correct phase boundary + +**Status: consistent with the design; no change requested.** + +Durable pins one immutable extension snapshot per phase. Registry table swaps are therefore safe for old/new code interleaving as long as extension values do not reach mutable/restarted natives behind the snapshot. The latter exception is exactly the code-mode and models-native problem above. + +#### Mixed facet versions made shared modules and local identities dangerous + +**Status: RESOLVED by all-or-nothing rollback per host.** + +The original design let a facet whose setup threw keep its prior version while other facets on the same host committed. That made copied Durable definitions and shared host behavior particularly dangerous. The owner later decided that any load or staging error rolls back the whole host; a host is entirely at revision R or R−1. + +Cross-host mixed revisions still exist during rollout and remain protected by versioned contracts and mismatch rules. + +#### TUI `instanceof` checks are not themselves a reload failure when slots are recreated + +**Status: resolved by the current TUI slot behavior; loader-D cross-plugin identity rules still apply generally.** + +The review considered `surface instanceof ModelState`, `PromptSurface` and `NoticeWatch`. The TUI runtime disposes and recreates a surface whenever its factory identity changes, so same-facet checks remain aligned after a successful table swap. This is not the same as importing a class from another plugin, which loader D correctly forbids. + +#### Web-shell reload remount cost is larger than presence alone + +**Status: OPEN as part of UI renderer reload.** + +Replacing the root component also: + +- remounts the app-scoped `ComposerState` and `NewSessionState`; +- churns every object hold below the root; +- can lose transient UI state not in browser/session/server storage; +- can reset scroll and sheet-local state; +- reruns global effects such as navigation listeners. + +Recommendation: keep the root shell identity stable and reload contribution tables below it, or explicitly snapshot/transfer all app-scoped state and remote holds. + +#### Prompt restart can turn a code reload into an expensive remote rescan + +**Status: OPEN as native split/ownership design.** + +Prompt watches may scan AGENTS.md, skills and templates over SSH, with a 60-second load timeout. If that watcher is in a native object, every reload of its owning plugin restarts the scan. Recommendation: keep the last valid prompt data and first-load completion in durable/facet documents, and isolate the expensive watcher into a narrowly changing owner/plugin if needed. + +#### Coding-tools section performed late registry lookup + +**Status: addressed conceptually by ownership principle; concrete replacement still to design.** + +`coding-tools/worker.ts:24` reads `HarnessEnvironments` at section-render time to obtain an SSH host. This treats an extension registry as a data service and lets a pinned Durable phase consult the current registry. Recommendation: the session/location owner should expose display location/SSH-host data; registries remain behavior extension points only. + +#### Models' stable proxy does not make captured model operations reload-safe + +**Status: still relevant to native ownership.** + +`Resources.models` is a proxy that binds a method lookup to the current immutable model generation (`models/resources.ts:80-86`). That correctly prevents one stream from switching provider implementation halfway through a call. It does not solve reload of the native owner, credential-file closure, or the fact that provider implementations contain plugin code. The p3 model runtime needs explicit generation pinning and shutdown semantics. + +#### Model and prompt native containers can pin code even without unbounded leaks + +**Status: RESOLVED mechanically by restart, but the distinction remains useful.** + +A kept class/native instance can pin its creation generation. This is often one permanently pinned old generation rather than unbounded growth, but its stale behavior is still wrong. Generation-GC tests must check both bounded reachability and behavior actually changing after reload. + +#### Registry values can become a loophole for “no natives across plugins” + +**Status: DECIDED in principle; enforcement remains a recommendation.** + +`HarnessConfig` showed that a registry entry can return `Models`, effectively moving a native across plugin boundaries while formally using a registry. Recommendation: static/API guardrails must enforce that registries add behavior and do not carry another plugin's native. Core-owned integration with Durable should not be disguised as a plugin registry. + +#### Raw timers in facets/native behavior need conversion or an explicit exception + +**Status: routines remain the decided mechanism; concrete call-scoped polling remains open.** + +Examples were logs' `setInterval`, PowerShell's retry sleep and code-mode's 25 ms `setTimeout` polling. Standing behavior belongs in routines. Polling internal to one call may need a call-scoped timer helper or watch rather than a standing routine. + +## Round 1 architecture recommendations (superseded sketches, retained for design intent) + +These were the intended ownership boundaries behind the original rewrite sketches. APIs such as `self.resource` are superseded, but the behavioral recommendations remain where they agree with p3 ownership: + +- models should expose data and operations (`hasModel`, defaults, enabled scope, login/logout/refresh) rather than an opaque `Resources` instance; +- prompt should keep loaded prompt data and failure state separately from the file watcher, and input gating should survive watcher/routine restart; +- code mode should settle its running Durable calls before a reload-driven sandbox close; +- coding-tools should read session-owned location metadata instead of using an environment registry as a query API; +- one-shot agent should not write subagents' private document; +- logs should own dedupe state outside restarted collection behavior; +- dev-tools and PowerShell should derive availability from session location through latest-wins reactions; +- web/TUI contributions should be registrations, but app/root state and holds must survive contribution reload. + +## Round 2: p3 ownership and native-object review + +The owner had already decided at this point: + +- **DECIDED:** p3 is a new, isolated app beside p2, with its own data directory and no legacy data migration. +- **DECIDED:** Chord v2 initially lives inside p3. +- **DECIDED:** natives exist only inside objects, opened during activation, fixed for the object's lifetime and closed at object end. +- **DECIDED:** facets are pure objects with no natives. +- **DECIDED:** load/staging failure rolls back the entire host. +- **DECIDED:** pure objects swap in place; native objects of reloaded plugins restart and their running work is aborted. +- **DECIDED:** each thing has one smallest owner; nobody reaches through it; registries add behavior and never guard owned data. + +The object list below was a **RECOMMENDATION**, not an owner decision. It was derived from the reviewed plugin interactions before the subsequent functionality-map exercise. + +### Recommended core ownership boundaries + +| Recommended type/address | Host | Kind | What it owns / why | +| --- | --- | --- | --- | +| `settings {}` | server | native file watch/lock | `settings.json`; validates schema, keeps last valid state, serializes writes | +| `credentials {}` | server | native locked file | `auth.json`; fail-closed parsing, metadata, token refresh/modification, no secret state exposure | +| `model-catalog {}` | server | native pi-ai model catalog/file watch | `models.json`, `models-store.json`, provider metadata, defaults/scope resolution, login/logout/refresh operations | +| `models {}` | server | pure per-owner view | owner-visible catalog/default/scope/errors derived from the catalog owner | +| `log-store {}` | server | native stdout capture/collector | bounded source buffers, notices and secret redactor | +| `logs {source}`, `log-notices {}` | server | pure owner views | query/filter state and dismiss operations over the log owner | +| `session {session}` | worker | native Durable Harness/SQLite/environments | session storage, location, set of conversations, relocation/admission/task lifecycle | +| `model-runtime {}` | worker/core | native `MutableModels` | synchronous model access required by Durable; request-generation pinning | +| `conversation {session, conversation}` | worker | pure owner over Durable data | entries, agent configuration and core policy (`readOnly`, `followsSessionCwd`, origin/creation metadata) | +| `chat {session, conversation}` | worker | pure per-principal view | authorized view/send/configure facade over the conversation owner | + +Recommended rules and APIs: + +- `settings`: expose validated state/errors and typed patches; plugins should not edit the file. +- `credentials`: expose non-secret metadata; worker-only credential resolution; serialized modify/delete. +- `model-catalog`: build/validate before publish; expose `has`, defaults, enabled scope, provider login/logout/refresh. +- `log-store`: enforce redaction, bounds and line-id dedupe; expose append/notice/dismiss/addSecret. +- `session`: enforce relocation fence/start barrier; expose location including `sshHost` and `os`, task abort, environment file/watch operations. +- `model-runtime`: expose model validation and choice resolution; keep its native inside core/Durable integration. +- `conversation`: enforce read-only/configure/send policy and model validity; expose `originOf(taskId)` instead of transcript parsing in unrelated plugins. + +### Recommended feature ownership + +| Area | Recommended owner/kind | Review recommendation | +| --- | --- | --- | +| models UI | pure facet/view | UI and command registrations separate from catalog/files native | +| prompt | worker native watcher plus facet documents | watcher writes loaded agents/skills/templates and first-load completion; keep last valid data across restart | +| code mode | worker native sandbox owner | owns sandbox and running call accounting; explicitly settles Durable work on reload close | +| coding tools | pure extension facet | section reads session-owned location data | +| one-shot agent | pure extension behavior | creates a child conversation with core policy/metadata in the creation commit | +| PowerShell | pure behavior plus boolean/document | derive from session-owned OS data; do not reach into environment connection native | +| dev control | server native dev channel plus worker extension | dev owner validates reload; worker tool delegates after conversation-origin authorization | +| logs UI | pure views | no native/log-store ownership in UI plugin | +| web shell | pure client facet | root identity/state still must survive renderer reload | + +### Places where p2 reaches past an owner + +Each row is a review finding and replacement recommendation. + +| Today | Replacement recommendation | Status | +| --- | --- | --- | +| `ModelConfig.resources` opaque used by login and sessions | model-catalog methods/state | **DECIDED principle; concrete owner open** | +| `HarnessConfig` carries native `Models` into Harness, chat, subagents and agent | core model runtime integrated with Durable | **DECIDED no cross-plugin natives; concrete split open** | +| auth.json written by server and workers | one credentials owner, or a deliberately versioned core file interface | **OPEN: config files with one writer explicitly listed** | +| settings.json read directly by every worker; keys consumed by unrelated features | settings owner with typed operations or sections | **OPEN: single-writer config listed** | +| agent imports `subagents/tools.ts` and writes `Subagents` | core child policy/metadata at creation | **DECIDED** | +| code mode calls native Harness APIs and polls `LiveDoc` to cancel work | session-owned task-abort operation | **RECOMMENDATION; close semantics open** | +| PowerShell reads `environments.at().connection.info()` | session/location owner exposes OS | **DECIDED principle** | +| prompt imports env-specific implementation and reaches native environment | session/environment owner offers file read/list/watch operations | **RECOMMENDATION** | +| coding-tools reads `.host` from `HarnessEnvironments` registry as data | session/location owner exposes `sshHost` | **DECIDED principle** | +| dev-tools copies `conversations/origin.ts` and parses transcript/task chains | conversation owner exposes origin/attribution operation | **RECOMMENDATION** | +| global `processLogs.redactor` is used directly by models, login and radius | log/redaction owner method or host value | **RECOMMENDATION** | +| `ReadOnlyConversations`, `SessionCwdFollowers` policy registries | core conversation policy document | **DECIDED** | + +### Natives recommended for isolation/splitting + +The p3 design says an expensive native should be split into its own plugin because all native objects of a reloaded plugin restart. These were the concrete candidates: + +| Native | Why restart hurts | Recommendation | Status | +| --- | --- | --- | --- | +| worker `MutableModels` | restarting during a stream can break auth/provider work and violates Durable's synchronous native needs | place in core/model-runtime rather than a frequently edited UI/config plugin | **OPEN concrete design** | +| server catalog and credentials/login | reloading models can abort a 10-minute OAuth flow after the user has acted | split stable native catalog/login ownership from pure models views/UI | **OPEN** | +| process log store | restart loses in-memory buffers, redactor knowledge and dedupe state | core/host native, independent of logs UI | **RECOMMENDATION** | +| code-mode sandbox | restart aborts long scripts | initially considered a separate stable plugin; final review judged a split alone insufficient because Durable needs the sandbox/run native interface | **OPEN; close semantics explicitly open** | +| prompt watcher | remote rescan can take up to 60 seconds | retain loaded data/first-load completion outside watcher; optionally isolate watcher if frequent code reload would restart it | **OPEN** | + +### What the ownership principle could not directly express, or made worse + +#### A. Durable requires native interfaces that are not data-returning methods + +**Status: OPEN / architectural constraint.** + +Durable calls model APIs such as `getModel` and streaming implementations synchronously through a `Models` object. Code mode calls `sandbox.start(code, { onCall, onIdle, … })`, receiving a run handle with delivery and result behavior. “Methods return data, never objects” cannot directly represent either native interface across an ownership boundary. + +Recommendations considered: + +- put model runtime in core beside the Durable Harness; +- keep code-mode extension and sandbox in the same native-owning object; +- do not disguise the native as a registry value; +- if a run must cross an object boundary, model it as an addressed run object rather than returning a native handle, but this is much heavier and was not recommended without broader need. + +The owner has decided that p3 gives Durable plain data and Durable's own types, never Chord v2 objects. The exact core/native integration remains to be designed. + +#### B. Native restart does not automatically abort Durable work + +**Status: OPEN; same as round-1 finding 2.** + +The owner decided native-object running work is aborted on reload, but Durable task execution is external work reached through an extension callback. Chord cannot infer which Durable tasks belong to the native. The native owner needs an explicit accounting/abort hook before close. + +#### C. Close needs a reason + +**Status: OPEN and explicitly listed in `redesign-decisions.md`.** + +> `close` with a reason (`reload` / `end` / `stop`): codemode must abort its Durable calls on reload, not on stop. + +This was the clearest unresolved API addition from this review. + +#### D. A single server credentials owner changes failure and trust boundaries + +**Status: OPEN; part of single-writer config design.** + +If every worker asks the server for credentials: + +- model work now depends on the server link; +- a server restart can park/fail token resolution during a running turn; +- refresh rotation takes a cross-process round trip; +- secrets cross the server-worker link. + +Alternative considered: workers have a core-owned, read-only credential-file reader while writes/refresh ownership remains serialized on the server. That makes the file format a versioned interface and weakens strict single-object ownership. This trade-off needs an explicit decision. + +#### E. One settings file conflicts with “smallest owner” + +**Status: OPEN; config files with one writer explicitly listed.** + +`settings.json` contains keys owned by models, Durable run behavior, code mode, owner identity and TUI/display behavior. Options: + +1. one settings owner knows all keys (risk: core god object); +2. plugins register versioned setting sections/keys (registry now influences a file owner's schema, but should not guard unrelated data); +3. split files by owner (behavior change for owner-edited configuration); +4. treat the file format itself as a core versioned interface and let one core owner persist it. + +No option was selected by this review. + +#### F. UI remount still releases authority/presence references + +**Status: OPEN; explicitly listed.** + +Same concrete login/presence/questions trace as round 1. Ownership does not solve lifecycle interpretation. + +#### G. Root/scalar document replacement was unspecified + +**Status: UNKNOWN/OPEN.** + +PowerShell availability and similar results are scalar values naturally replaced as a whole. If documents only expose mutable object drafts, the API needs root replacement (`tx.set`/`handle.set`) or these values must be wrapped in objects solely to mutate a property. + +#### H. Call-scoped natives may be needed + +**Status: OPEN and explicitly listed by the owner as raised by reviewers.** + +Some natives exist for exactly one call rather than one object lifetime. The reviewed examples suggest provider login interactions, transient subprocess/run handles and some environment operations. Forcing each into a long-lived object or a data-returning method can distort ownership and cancellation. Chord v2 may need a call-scoped native mechanism, but this review did not settle its API. + +#### I. Async activation guards for authority-bearing addresses + +**Status: OPEN and explicitly listed by the owner as raised by reviewers.** + +If an address itself carries authority or requires asynchronous validation, checking only after an object becomes live creates a window with the wrong lifecycle/presence semantics. The runtime may need an activation guard that completes before publication. No concrete API was selected here. + +#### J. Invariants spanning owners + +**Status: OPEN and explicitly listed by the owner as raised by reviewers.** + +The ownership principle deliberately avoids cross-owner transactions, but some app rules span several owners' data. This review encountered examples around session/conversation creation metadata, credential/catalog coordination, and reload/worker state. The rule must identify which owner is actually smallest for the invariant, or define a coordinator/protocol; policy registries are not an acceptable guard. + +#### K. Published projections for readers that must not start workers + +**Status: OPEN and explicitly listed by the owner as raised by reviewers.** + +Logs, session status, unread/child summaries and similar server-side views must be readable without activating a session worker. Copies need a generation fence so an older worker cannot overwrite newer server state. This review's logs and worker-failure traces are concrete examples. + +#### L. Connection pooling + +**Status: OPEN and explicitly listed by the owner as raised by reviewers.** + +Environment consumers can independently open SSH/native connections. The ownership model needs a connection-pool owner or another way to avoid four parallel SSH connections while still preventing plugins from reaching through the environment owner. + +### Earlier findings after the p3 redesign + +This was the review's explicit resolution table, updated only with known owner status: + +| Earlier finding | Result under p3 redesign | +| --- | --- | +| Kept resources keep running old logic | **RESOLVED:** native objects restart with new code. | +| Code-mode sandbox hang | **REMAINS, and can happen on every code-mode plugin reload:** needs task settlement and close reason. | +| Client remount releases references | **REMAINS / OPEN.** | +| Routines `overlap: "latest"` | **REMAINS / OPEN.** | +| Agent writes subagents' document | **RESOLVED / DECIDED** by core conversation policy at creation, provided one-shot listing metadata also gets one proper owner. | +| Derives over a native's replacement/change source | **RESOLVED mechanically** for native replacement because a native is fixed for the object lifetime; native-internal change still needs an explicit readable. Whole-root document replacement remained unspecified. | +| Natives across plugins (login/models) | **RESOLVED as a rule:** forbidden. Concrete ownership and login-restart cost remain open. | +| Logs dedupe and per-line cost | **REMAINS / OPEN.** | +| Mixed old/new facet versions after one setup rejects | **RESOLVED:** staging failure now rolls back the whole host. | +| Old module generations retained by resources | **RESOLVED by native restart**, subject to external Durable/registry closures not retaining them. | + +## Final recommendations carried forward + +These are the recommendations from both rounds that remain applicable to Chord v2/p3: + +1. **Specify native close reasons.** At minimum distinguish reload, ordinary object end/removal and process/worker stop. Code mode is the concrete acceptance test. +2. **Define ownership of external work.** A native owner must be able to enumerate/abort/settle Durable tasks or other non-Chord work it started before close. +3. **Add `routines.on(..., { overlap: "latest" })`.** Relocation-sensitive prompt and OS detection are acceptance tests. +4. **Make renderer replacement invisible to presence and authority leases.** Transfer equivalent holds across web remount/TUI worker swap before releasing the old client. +5. **Keep core policy with the conversation.** `readOnly`, cwd-following and one-shot origin/metadata are written in the child-creation commit; never restore policy registries or cross-plugin private-document writes. +6. **Keep registries behavioral.** Do not use registry values as native/data-service loopholes (`HarnessConfig`, environment host lookup). +7. **Choose and document single-writer strategies for `auth.json`, `settings.json` and `models.json`.** Include server-restart, worker-offline, token-refresh and owner-hand-edit traces. +8. **Place synchronous model runtime beside Durable/core.** It cannot cleanly cross a plugin boundary under “methods return data”. Preserve per-request generation pinning. +9. **Define root document replacement.** Add a typed whole-value set operation unless every state schema is required to be a mutable object. +10. **Keep expensive watcher results outside the watcher native.** Prompt's last valid data and first-load completion must survive watcher restart; models/config similarly need explicit published state. +11. **Centralize log ownership and redaction.** Preserve dedupe state across UI/collector reload; publish append/trim operations rather than full-list diffs under load. +12. **Test behavior change, not only generation GC.** A reload test should prove fixed model/prompt/log logic executes afterward, old code is collectable, running native work gets the specified outcome, and failed staging leaves the complete host at the prior revision. +13. **Use bundle dependency metadata or ban copied behavioral definitions.** Even with host-atomic rollback, targeted reload must not leave copied Durable/task/document definitions stale in another plugin. +14. **Keep the root client shell stable where possible.** Reload contribution tables below it instead of remounting every screen, object hold and app-scoped editor state. +15. **Treat the ownership principle as an enforcement rule, not prose.** Static/runtime checks should catch opaque/native returns, cross-plugin private document definitions/writes, and registries used as data/policy guards. diff --git a/apps/p2/docs/p3-reviews/pl-chat.md b/apps/p2/docs/p3-reviews/pl-chat.md new file mode 100644 index 000000000..5bb1ec338 --- /dev/null +++ b/apps/p2/docs/p3-reviews/pl-chat.md @@ -0,0 +1,346 @@ +# pl-chat: target design, reload, Chord v2 and p3 reviews + +This records the reviews, findings, questions and recommendations produced in this conversation, excluding the functionality map. The two review reports below are preserved verbatim apart from their heading levels. They are historical reports, not a claim that every proposal still applies. Additional review points from the discussion follow them. + +## Decision and open-status index + +Known owner decisions come from `packages/chord/docs/decisions.md` at `5abe3f6d6`. Proposed object boundaries and fixes below are recommendations, not owner decisions. + +| Point | Known status | +|---|---| +| Build p3 next to p2, isolated data, no legacy migration; Chord v2 inside p3 initially | Owner decided | +| Pure facet, natives only in served objects through `self.open`; no resources, replace or deps | Owner decided; supersedes the first report's resource proposals | +| Native objects restart and abort running work on reload; expensive natives split into separate plugins | Owner decided | +| Whole-host rollback for load/staging errors, including live-object factories | Owner decided; resolves first report F1's setup-throw version mix | +| Named derive with its previous value retained across factory re-runs | Owner decided; resolves F3's missing mechanism | +| Jobs retain intent/run/finish and add awaitable `outcome(ctx)` | Owner decided; job lifetime and dedupe-retention questions remain open | +| Ownership principle; methods return data, objects passed as addresses; no cross-plugin natives | Owner decided | +| Conversation policy written in the creating commit; remove external read-only/cwd-following policy registries | Owner decided | +| Registries remain behaviour extension points, including send routers and input gates | Owner decided | +| Single-shot Durable watches | Mechanism changes resolve retained-watch reuse; proposed split into a separate feed object/plugin remains a recommendation | +| Draft live representation, withdrawal/restoration atomicity and retention | Open recommendations; no known owner decision adopting this review's particular design | +| SessionDeleted, file formats, optional references, UI renderer reload versus leaving, shared Durable docs | Explicitly open in redesign-decisions.md | +| Keep-alive calculated by worker; presence/link-drop/linger details | Explicitly open in redesign-decisions.md | +| Proposed Catalog, Session, Conversation, Composer, Downloads and Artifacts boundaries | Review recommendations, not adopted decisions | +| Cross-owner chat digest concurrency, byte transfer overhead, relocation environment addresses | Open review questions/recommendations | +| Precise migration estimates in report 1 | Historical only: superseded by the decision to build p3 without legacy migration | + +## Original review against target-design.md and plugin-loading.md, 6559589bd + +I read target-design.md, plugin-loading.md and questions.md in full on origin/private `6559589bd`, all 31 files of the six plugins, and the runtime they use (`worker.ts`, `durable.ts`, `commits.ts`, `contribute.ts`, `chord.ts`, `tui.ts`). I also checked pi-durable's `WatchHandle`. Nothing was edited; the only scratch files are under /tmp/plrev. + +**Verdict:** the design holds for these six plugins, but five gaps need fixing first. conversations, send-file and the artifacts server, web and TUI facets port almost unchanged. The problems are all in chat, chat-mode, drafts and the artifacts worker. + +### Sketches in the new design + +```ts +// chat/worker.ts +export default defineFacet((host, self) => { + const harness = host.value(HarnessKey), envs = host.value(EnvironmentsKey), liveness = host.value(LivenessKey); + host.serve(Conversation, ({ conversation }, _p, self) => { + const id = conversation as ConversationId; + const conv = self.resource("conversation", { version: 1, + open: async ({ signal }) => (await harness.conversation(id, durableContext(signal))) ?? fail(`Unknown ${id}`) }); + const view = self.source("view", { deps: [conv], // MISSING API, F2 + open: ({ signal }) => watchSource(conv.value.watch(durableContext(signal))) }); + const window = self.derive([view], (v, prev) => v && windowView(v, prev)); // needs prev, F3 + const model = self.derive([view, host.registry(HarnessConfig).table], (v, t) => modelMeta(v, t)); + const last = self.state(LastShell); // singleton: {id, data} | null + const shell = self.job(ShellJob, { concurrency: 1, + run: async ({ command, includeInContext, author }, ctx) => { + const agent = await conv.value.agent(durableContext(ctx)); + const out = new OutputBuffer({ maxBytes: SHELL_MAX_BYTES, maxLines: SHELL_MAX_LINES, retain: "tail" }); + const r = await envs.at(agent.cwd).exec(command, { ...opts, + onOutput: (t, _c, i) => { out.push(t, i.skipped); ctx.progress({ output: out.snapshot(), truncated: out.truncated }); } }, + durableContext(ctx.signal)); + const data = shellData(command, author, includeInContext, out, r, ctx.signal.aborted); + await conv.value.submit({ type: "write", entry: shellEntry(data), requestId: `${author.id}:shell:${ctx.jobId}` }, + durableContext(ctx)); // admission only, no .wait (F4) + return data; + }, + finish: async (done, tx) => { (await tx.draft(last)).value = { id: done.id, data: done.ok ? done.result : errorData(done) }; }, + }); + self.routines.on(view, (v, ctx) => last.value && placed(v, last.value.id) && last.change(ctx, (d) => { d.value = null; })); + return { + view, window, model, + shell: self.derive([shell.running, last], liveShellOf), // ShellState | null, as Chat v5 + runShell: async ({ requestId, author, ...a }, ctx) => { + for (const gate of host.registry(InputGates).table.value.values()) await gate(ctx); + return { shellId: (await shell.start({ ...a, author }, ctx, { key: `${author.id}:${requestId}` })).id }; + }, + abortShell: ({ shellId }) => ({ aborted: shell.abort(shellId) }), + submit, write, stop, dequeue, dequeueOwn, history, configure, compact, reset, abortCompaction, costs, // unchanged + }; + }, { linger: 30_000 }); + host.serve(Chat, (input, principal, self) => { /* as today, minus the "viewer" resource (-> host.references); + call = liveness.call */ }); + host.serve(ChatCosts, /* unchanged */); +}); +// chat/contract.ts: SendRouters / TranscriptEntries get { version: 1 }; Conversation unchanged. +// chat/web.tsx: self.register(Slots, "main", Main) +// chat/tui.ts: self.register(TuiSlots, "chat", (rt) => new ChatSurface(rt)); +// for (const [k, c] of Object.entries(CHAT_COMMANDS)) self.register(TuiCommands, k, c); +// ChatSurfaceKey = defineSurface<ChatApi>("chat") (14.3) +``` + +```ts +// chat-mode/worker.ts +export default defineFacet((host, self) => { + const harness = host.value(HarnessKey); + host.serve(ChatModeSetting, ({ conversation }, _p, self) => { + const doc = self.source("doc", { open: ({ signal }) => + watchSource(harness.watchDoc(ChatModeDoc, conversation, durableContext(signal))) }); // replaces CommitSignal + task + const view = self.get(Conversation, { conversation }).view; + return { + enabled: self.derive([doc], (d) => d?.enabled ?? false), + waiting: self.derive([view], waiting), // once per conversation, F8 + set: ({ enabled }, ctx) => commitChatMode(harness, conversation, enabled, ctx), // Durable doc, as today + }; + }, { linger: 30_000 }); + self.register(SendRouters, "chat-mode", routeChatMode); // body unchanged + host.serve(ChatMode, (i, p, self) => { const s = self.get(ChatModeSetting, i); + return isOwner(p) ? { variant: "owner", enabled: s.enabled, waiting: s.waiting, set: s.set } + : { variant: "reader", enabled: s.enabled, waiting: s.waiting }; }); +}); +// web: self.register(Slots, "composer:chat-mode", ChatModeChip); self.register(TranscriptEntries, k, v) for each display +// tui: same registrations; chat.setCanReply via ctx.surface(ChatSurfaceKey) + +// drafts/worker.ts: one persisted singleton (F5), not keyed +const DraftsDoc = defineState("drafts", { version: 1, persist: true, schema: Type.Object({ next: Type.Number(), byKey: DraftsByKeySchema }), + initial: () => ({ next: 1, byKey: {} }) }); +host.serve(DraftStore, (_i, _p, self) => { + const doc = self.state(DraftsDoc); + return { + drafts: self.derive([doc], (d) => d.byKey), + add: ({ conversation, items }, ctx) => self.tx(ctx, async (tx) => { const d = await tx.draft(doc); + for (const { author, text } of items) (d.byKey[draftKey(conversation, author.id)] ??= []).push({ id: d.next++, text }); }), + claim: ({ conversation, authorId, ids }, ctx) => self.tx(ctx, async (tx) => { /* filter, delete key when empty, return texts */ }), + }; +}, { lifetime: "host" }); +// Drafts (per author): self.derive([store.drafts], (all) => all[key] ?? []) unchanged + +// conversations: server unchanged (Hosts -> host value); worker: self.register(Extensions, "p2-conversations", extension); +// PeerConversation uses host.value(LivenessKey).call; web: self.register(Slots, "chat-attribution", Attribution) +// send-file/worker: self.register(Extensions, "p2-downloads", createSendFileExtension(host.value(WorkerInitKey).dataDir, session)) +// (the template-plus-late-execute trick goes away); SentFiles unchanged; tui: renderers + command, tool views via ChatSurfaceKey +// artifacts/worker: Extensions as send-file; ArtifactList: self.derive([self.source("doc", watchDoc(ArtifactDoc, id))], d => d?.artifacts ?? []) +// artifacts/server: const dir = host.value(ServerInfoKey).dataDir; +// self.register(Routes, "GET /artifact", serve); self.register(Routes, "HEAD /artifact", serve) +// (the "artifacts" resource is not a native: drop it) +``` + +### Findings, most severe first + +**F1. Two plugins on different versions in one host can write wrong history; I challenge the decision on this.** The decision assumes "mixed versions within a host are the same situation as during a rollout". That is not true for `local` types: they never cross hosts, and they are not validated. +- Trace: commit `e16667441` added `maxImages` to `Conversation.submit` and chat-mode's use of it in one change, with no version bump (`chat/contract.ts:170`, `chat-mode/digest.ts:173`). Commit `79c5e20d5` added `entry` the same way (`contract.ts:166`). +- Suppose chat's worker setup throws on such a reload. Chat keeps its old code; chat-mode moves on, and its router calls the old `Conversation.submit` with `entry: chat.mention` (`chat-mode/worker.ts:164`). +- The coverage check admits it: an extra optional property is "covered" by an open TypeBox object. Old `submit` drops `entry` and writes a `p2.input` entry whose text is the digest, without `included`. +- Durable entries are permanent. `unmentioned` (`digest.ts:68-84`) never finds a boundary again, so every later mention re-quotes up to 100 messages. With `maxImages` dropped, every image of the digest is inlined. +- Fix: for a facet setup that throws, also reject the facets that hold that plugin's local types in the same revision on that host. Alternatively, make the coverage check require the provider to declare every property the caller's arguments declare, so this fails with `contract_mismatch` instead of being dropped silently. + +**F2. There is no declaration for a source backed by a subscription, and Durable watches work only once.** +- A Durable `WatchHandle` has one `start` and a terminal `stop` (`durable/src/types.ts:902-904`). `watchSource` allows one subscription (`runtime/durable.ts:112-121`). +- Chat holds the watch as a resource and subscribes to it (`chat/worker.ts:158-176`). Under §12/§15, a reload keeps the resource (same version) and re-runs the factory and routines. +- The new subscription throws "supports one subscription", or the old unsubscribe has already called `watch.stop()`. Either way every transcript freezes after a chat reload. +- The pattern `CommitSignal` uses in chat-mode and artifacts (`chat-mode/worker.ts:88`, `artifacts/worker.ts:62`) attaches a handler inside resource `open`, which 14.5 forbids. §8 names "source readables via derive" but gives no API for one. +- Fix: add `self.source(name, { deps, open })`. It acquires the native per subscription and is restarted with routines. The derive keeps its last value until the new source delivers. Add a p2 `watchDoc` helper on top of it. + +**F3. Derives have no previous value, and a reload resets the chat window for every viewer.** +- `windowStart` keeps the previous start within a 1.5× slack (`chat/window.ts:32-44`, using the `shown` memo at `worker.ts:167`). That is why new entries are small appends rather than a window shift on every commit. +- A pure `derive(sources, fn)` cannot express this. A closure memo is lost on every factory re-run, and so is the module-level `entrySizes` cache (`window.ts:17`). +- After a reload, each live conversation re-stringifies its whole active range (MBs on long sessions), possibly inside the one-tick publish. Every viewer then gets a window that may have moved, up to 512 KiB each. +- Fix: `self.derive(sources, (values, previous) => …)`, keyed by derive name so `previous` survives factory re-runs. + +**F4. Shell as a job: object lifetime, output display and deduplication are unspecified.** +- **Lifetime:** `Conversation` is a referenced object with `linger: 30_000` (`chat/worker.ts:505`). Today, closing the object aborts the shell (`:186-195`). The design does not say whether a running job keeps its object alive. If it does not, `finish` has no object once the last tab has been closed for 2 min plus 30 s. +- **Display:** today the finished output stays visible until the `p2.shell` entry is placed (`:290-322`). When the agent is busy, that happens at the run's next boundary, which can be minutes away. If `run` waits for placement, `concurrency: 1` refuses the next `!cmd` with SHELL_BUSY. If it does not wait (§10's example), the output disappears until the entry appears. +- **Deduplication:** today the dedupe map keeps 100 keys (`:333-341`). The design does not say how long a finished run's key is kept, and §21 asks for "shell dedupe survives". +- Fix: running or finishing runs count as a reference to their object. Add `dedupe: { keep: N }` to `defineJob`. `liveShell` derives from `running` plus a singleton `LastShell` document, which a routine on `view` clears once the entry is placed (sketch above). + +**F5. The drafts plan in §21 contradicts §7, and Stop is not atomic across Durable and Chord.** +- §21 says drafts are "persisted, keyed `conversation:authorId`". But keyed documents have no live readable, and `Drafts.drafts` must be live: web (`chat/web.tsx:362-370`), TUI (`tui.ts:340-348`), derive (`drafts/worker.ts:49`). +- Stop withdraws inputs in a Durable commit (`chat/worker.ts:435-441`), then writes other authors' texts in a separate Chord commit (`:537-541`). A crash in between loses exactly the text that persistence was meant to keep. +- Fix: a persisted singleton map (sketch above, a 1:1 port), plus a retention routine. Guests' unclaimed drafts otherwise grow forever. Optionally record the restore intent first, as a job (intent, then abort, then finish). + +**F6. Session delete leaves plugin files behind.** +- Artifacts are stored under `<data>/artifacts/<session>` (`artifacts/store.ts:111`) and sent files under `<data>/downloads/<session>` (`files/downloads.ts:57-60`). +- Clone already has to hard-code these directories, plus uploads (`sessions/copy.ts:32`). §7's delete covers only tagged documents, the session database, the document storage and the catalog entry, so these files, which may contain secrets, stay on disk. +- Fix: a versioned server registry `SessionData`, where plugins declare per-session directories or cleanup. Both clone and delete use it. + +**F7. A TUI facet reload drops chat state and closes unrelated overlays.** +- A `TuiSlots` swap disposes and recreates the surface (`runtime/tui.ts:473-478`). `ChatSurface` keeps uploaded `attachments` (`chat/tui.ts:376`), `replyTo`/`canReply` (`:87-89`) and `sending`/`waiting` on itself. +- After a chat TUI reload, the next send goes without the uploaded file. It may even reuse the same `requestId` from `runtime.submissions`, so the server deduplicates it as the earlier message. +- Any reload that changes the TUI command table also closes every overlay (`tui.ts:463-466`). +- Fix: move attachments and reply state into `TuiRuntime` and its swap snapshot, like drafts. Close only overlays opened by entries that changed. + +**F8. Performance: chat-mode's waiting count is recomputed per viewer.** +- `ChatMode` is remote, so there is one object per principal, and each has `derive([chat.view], waiting)` (`chat-mode/worker.ts:173`). Each run reverses the whole view and scans it (`digest.ts:110`). +- That means viewers × entries work on every streaming commit. `CommitSignal` also runs its filter for every object on every harness commit. +- Fix: compute `waiting` once in the per-conversation `ChatModeSetting` (sketch above) and replace `CommitSignal` with `watchDoc` sources. + +Checked with no issue found: conversations (broker retry, Durable request-ID dedupe, rollout parking), keep-alive and presence (the viewer resource becomes `host.references` plus the linger bit), Durable phases pinning `Extensions` tables, and natives (all become host values). The `instanceof` surfaces are already covered by 14.3. + +### Migration cost + +About 17 plugin files. +- **chat:** `worker.ts` about 150 lines touched (shell becomes a job, net about −40), `contract.ts` about 5, `tui.ts` about 25, `tui-commands.ts` about 5, `web.tsx` about 3. +- **chat-mode:** `worker.ts` about −50/+20; `tui.ts` and `web.tsx` about 10. +- **drafts:** `worker.ts` about 35. +- **conversations:** about 15 across three files. +- **send-file:** about 20. +- **artifacts:** about 40 across four files. +- **Total:** roughly 350–450 lines touched, net about −100. +- **Runtime:** about 80 lines for `self.source`/`watchDoc` and `TuiRuntime` attachments, plus whatever Chord needs for F2, F3 and F4. + +**Does the design hold?** For these plugins, yes, once F1 (the setup-throw rule or the coverage check), F2 (`self.source`), F3 (`previous` in derive) and F4 (job ↔ object lifetime and key retention) are added, and §21's drafts entry is fixed. F6 is a privacy gap in §7, not a design flaw. + +## Original ownership review after redesign-decisions.md, 5abe3f6d6 + +### p3 objects for chat, chat-mode, drafts, conversations, send-file and artifacts + +I read `redesign-decisions.md` at `5abe3f6d6`. Two things change the shape of these plugins. First, a native object restarts on reload, which aborts the work running in it. Second, the ownership principle removes three things: the drafts plugin, the Durable chat-mode document, and files that two processes share on disk. + +### Core objects + +| Object `{address}` · host · kind | Owns · rules | API · used by | +|---|---|---| +| `Catalog {}` · server · pure | Session rows as persisted docs (title, env, cwd, archivedAt, a copy of the root model, scope). Rules: lifecycle order; relocate only under the worker fence; clone and delete call the owners' copy/delete hooks | `sessions`, `get`, `create`, `archive`, `restore`, `relocate`, `fork`, `clone`, `delete`, `rootChanged` (reported by Session) · `Sessions` views, broker, routes, chat web | +| `Sessions {}` · server · pure, one per principal | Nothing. Shows the catalog filtered for owner, guest or worker | session list, owner actions · web, TUI | +| `Access {}` · server · pure, one per principal | The grant (shares plugin) | `grant` · chat web, TUI | +| `Hosts {}` · server · native (worker processes) | Worker processes and their generations. Rule: one current generation per session | `current(session)`, start, stop · broker, status | +| `Session {session}` · worker · native (Durable harness, sqlite, scheduler) | The Durable store, the set of conversations, the location, the admission fence. Rules: a conversation's policy doc (`readOnly`, `followsSessionCwd`) is written in its creating commit; relocate only when fenced | `location`, `createConversation(init)`, `prepare`/`cancel`/`relocate`, `inspect` · Conversation, subagents, server lifecycle | +| `SessionEnv {session, location}` · worker · native (daemon or SSH connection) | Execution environments by cwd. Rule: never swapped; a relocation means a new address | `exec`, `at(cwd)` (local type, may take callbacks) · Session (Durable's env), Conversation shell, file index | +| `ConversationFeed {session, conversation}` · worker · native (Durable watch) | Nothing | `view` (complete) · Conversation only | +| `Conversation {session, conversation}` · worker · pure, local, `storeKey` | Its Durable content (entries, inbox, `pi.*` docs) through Session; the policy doc; a restored-drafts doc (persisted, by author); `ShellJob` runs plus a `LastShell` doc. Rules: input gates; read-only refusal; one request per author `requestId`; one shell at a time; only known models; withdrawn text goes back to its author | state: `view`, `window` (derive with `previous`), `model`, `liveShell`, `canReply`. methods: `submit`, `write`, `stop`, `dequeue`, `dequeueOwn`, `history`, `scan`, `entry`, `queued`, `latestAnswer`, `configure`, `compact`, `reset`, `abortCompaction`, `runShell`, `abortShell`, `costs`, `claimRestored` · Chat, chat-mode router, PeerInbox | +| `Chat {session, conversation}` · worker · pure, remote, one per principal | Nothing. Rules: canView/canSend; only the owner configures or resets; only the author or owner aborts a shell; stamps the author; runs `SendRouters` | v5 state plus `restored` (own texts only) and `canReply`; v5 methods plus `claim`, `costs` · web and TUI chat | +| `Composer {session, conversation}` · client · pure | Draft text, attachments, `replyTo`, pending request ID | get/set · chat surface, chat-mode chip | + +### Plugin objects + +| Object · host · kind | Owns · rules | API · used by | +|---|---|---|---| +| `ChatMode {session, conversation}` · worker · pure, local (chat-mode) | The switch as a persisted Chord doc (it is not tied to the transcript). Rules: only the owner switches; how a mention is composed | `enabled`, `waiting` (computed once per conversation), `set`. Registers the `SendRouters` entry "chat-mode", which also declares reply support. Remote face per principal: reader or owner | +| `ConversationBroker {}` · server · pure (conversations) | Nothing. Rules: sender and recipient not archived, no sending to yourself, current generation (asks `Hosts`), one retry | `query` · `Conversations` (worker face) | +| `PeerInbox {session}` · worker · pure (conversations) | Nothing. Rule: callable only by the server's broker | `send` (calls `Conversation(1).submit` with author `{kind:"peer"}`), `answer` (calls `latestAnswer`). Registers the `conversations` tool | +| `Downloads {session}` · server · pure (files) | `<data>/downloads/<session>` plus an index by conversation. Rules: 25 MiB cap, immutable IDs, viewers only | `save` (chunked), `list({conversation})`, `preview`, `GET /download`, `copy`, `delete` · send_file tool, route, TUI `/downloads`, web card | +| `Artifacts {session}` · server · pure (artifacts) | `<data>/artifacts/<session>` plus a per-conversation index doc. Rules: name, type and size checks, numbered immutable versions, CSP | `save` (writes file and index in one owner step), `list({conversation})`, `read`, `/artifact` route, `copy`, `delete`. Remote face `ArtifactList` · tool, web, TUI | + +Keeping `Downloads` and `Artifacts` on the server means listing, previews and routes no longer start a worker. + +### (a) Where today's code reaches past an owner + +| Today | Replacement | +|---|---| +| chat-mode reads `ChatModeDoc`, the inbox and entries straight from the harness (`chat-mode/worker.ts:78-100,135-151`) | `ChatMode` doc; `Conversation.scan`, `entry`, `queued` | +| `PeerConversation` submits to the root itself and copies the gate loop (`conversations/worker.ts:133-149`); `answer` scans entries itself (`:152-183`) | `Conversation.submit` with a peer author; `latestAnswer` | +| Peer attribution is a `<mobile-conversation>` text prefix that `originOf` parses (`origin.ts:47-60`) | Peer input written as `p2.input` with a peer author, a format core owns (`input.ts`) | +| `DraftStore` is a separate plugin that Chat writes into (`chat/worker.ts:537-541`) | Restored-drafts doc in `Conversation` | +| send-file and artifacts write data files directly through the copied `files/downloads.ts` and `store.ts`, using the opaque `worker.init.dataDir` (`send-file/worker.ts:33,45`, `artifacts/worker.ts:25,37`) | `Downloads.save` and `Artifacts.save` methods | +| Clone copies a hard-coded list of plugin directories (`sessions/copy.ts:32`); delete has nothing | The owners' `copy`/`delete` hooks (registry `SessionData`) | +| chat-mode sets chat's UI state: `setCanReply`, `instanceof ChatSurface` (`chat-mode/web.tsx:636-642`, `tui.ts:62-63`) | `Chat.canReply` plus the `Composer` object | +| `/downloads` reads ChatSurface's tool views (`send-file/tui.ts:93-103`) | `Downloads.list` | +| chat calls `Sessions.configureSelf` for the root model (`chat/worker.ts:622-629`); `ConversationConfigured` registry | Session reports root agent changes to `Catalog.rootChanged` | +| Opaque natives used by chat: `HarnessConfig.models()` (`worker.ts:101,135`), `harness.environments`, `liveness` | `WorkerModels.model(ref)` (data), `SessionEnv`, Session's fence | +| `ReadOnlyConversations`, the `viewer` liveness resource | Policy doc (already decided); `host.references` | + +### (b) Natives to split into their own object + +| Native | Why | +|---|---| +| Durable watch, as `ConversationFeed` | If it stays inside `Conversation`, a chat reload restarts the object and aborts a running `!cmd` (a ShellJob). Split out, `Conversation` stays pure and its jobs and docs survive. | +| Harness and `SessionEnv` | A restart aborts every task and the SSH connection. They stay in core; only a base change (host restart) touches them. | +| Durable `ChatModeDoc`, `ArtifactDoc` watches | Not split: the data moves to Chord docs, so no watch is needed. | + +### (c) What the principle cannot express or makes worse + +1. **A rule whose data has another owner.** Two people send `@pi` at the same time. Both routers compute the same unmentioned set (`chat-mode/worker.ts:150-151`) before either mention is in the inbox, so both digests quote the same messages. chat-mode owns the rule, but `Conversation` owns the inbox. Fix: `Conversation.submit({ expect: inboxSeq })` as a compare-and-set, or let routers run on the conversation's line. +2. **Bytes through methods.** Today the tool writes the download directory directly. With one owner, the 25 MiB snapshot (`send-file/tool.ts:68-78`) travels from worker to server in 1 MiB chunks: base64 adds a third and needs memory on the phone. The wire size limit is still open. +3. **One owner, two stores.** `Conversation` owns both the inbox (Durable) and the restored drafts (Chord). Stop's abort and the drafts write are still two commits (`chat/worker.ts:435-441`, then `:537-541`); a crash in between loses the text. Fix: write an intent in a Chord tx first, then abort, then finish, as a job. +4. **Durable tools cannot use Chord objects.** `originOf` runs inside a Durable tool and must read Conversation entries itself (`origin.ts:31-71`). The core-owned input format keeps this to reading one shared format. +5. **Relocation swaps a native in place today** (`Execution.switch`, `runtime/worker.ts:431-451`), and the new rules forbid that. It becomes a new `SessionEnv` address per location, which works only because relocation requires that nothing is running. + +### (d) Earlier findings + +| Finding | Status | +|---|---| +| F1 mixed versions after a setup throw | Resolved: rollback is all-or-nothing per host | +| F2 one-shot watches, no source declaration | Resolved: `self.open`. It costs a restart, so the watch moves into `ConversationFeed` (b) | +| F3 derive without `previous` | Resolved. Still open: the module-level `entrySizes` cache (`window.ts:17`) is rebuilt on every reload (minor perf) | +| F4 jobs | Partly. `outcome()` was added. Still open: a job keeping its referenced object alive (`linger: 30_000`), how long dedupe keys are kept, and keeping the output visible until the entry lands (`LastShell` doc) | +| F5 drafts | The keyed-document conflict is gone (restored drafts become a doc in `Conversation`); atomicity remains, see (c)3 | +| F6 session delete leaves files | Open, listed in Open as `SessionDeleted`; proposed owners' hooks above | +| F7 TUI surface state lost on reload | Fixed by the client `Composer` object; the overlay closing stays open as "renderer reload" | +| F8 waiting computed per viewer | Resolved: `ChatMode` is per conversation | + +## Additional discussion points and qualifications + +These points were raised while investigating or sketching the reports above. They are retained here so the record includes recommendations and questions that did not become one of the final numbered findings. Where an item overlaps a report, the report gives the main trace. All are open recommendations unless their status is explicitly stated otherwise. + +### Reload, subscriptions and derived values + +- Original watch alternatives considered: acquire the watch inside `self.routines.custom`, or add a lifecycle-aware source helper that reacquires on every subscription. Opening a fresh watch after the old handler settles can catch up from the latest snapshot while readers keep the previous value. A permanently kept single-shot watch cannot be restarted merely by giving it a new wrapper. Status: superseded by `self.open` and native-object restart, both owner decisions; a separate feed remains a proposed boundary. +- The `CommitSignal` container subscribes to harness commits in its constructor, called inside resource `open`. That is behaviour inside a retained native. Logic changes to its predicate would otherwise stay in the old generation. Status: old resource concern superseded; prefer Durable document watches or move the app state into Chord docs where appropriate. +- Chat-mode's `enabled` and `waiting` are derived from separate readables, so a client may briefly see inconsistent values if their updates are not one atomic publication. This was considered a minor chip flicker, not a new blocking finding. Status: open wire/publication consideration. +- A fallback for window hysteresis was an in-memory document holding the window start rather than a reducer-style derive. Status: owner chose named derives retaining the previous value. +- Even with previous-value derives, rebuilding `entrySizes` on reload can still cause work proportional to the active transcript size. The original concern was JSON serialization and a possible large window diff for every viewer; not every derive necessarily runs during publish, so the synchronous timing needs measurement. Status: remaining performance question. +- Module-level caches, such as entry-size caches, must not be used for cross-facet identity. Copied pure modules are fine if their state is only a disposable optimization. Status: loader-D contract/pure-module restrictions were already decided in plugin-loading.md. + +### Shell jobs, crash handling and lifetime + +- The job's raw `running` array is not the old `ShellState | null` contract. An adapter derive must preserve the old UI fields (author, command, output, truncated, running, exitCode, cancelled, error). This is implementation work, not a reason to expose internal job records directly. +- A command and its subsequent Durable placement wait are different kinds of busy. Holding the concurrency slot through `submission.wait()` closes a race but changes today's ability to start another command as soon as execution ends. Preserve the UI semantics explicitly rather than treating every unsettled run as an executing process. +- The original design's keyed `ShellResult` documents can accumulate indefinitely: in-memory docs belong to an identity rather than just a live instance, and the example never retires them. Either retire results after placement or keep only a bounded singleton result plus a bounded dedupe history. Status: open retention policy. +- Persistent job keys must remain scoped to the stamped author (`${author.id}:${requestId}`), not only the client-supplied request id. The dedupe record must outlive a lost admission response sufficiently to stop a retry from rerunning a destructive command. Status: open lifetime/retention policy, existing author attribution remains required. +- The prototype sketch used a single `LastShell`. It was a recommendation for bridging the gap until transcript placement, not a completed design for arbitrarily many finished-but-unplaced commands. Status: open implementation detail. +- The existing shell catches execution errors and keeps partial output, strips one trailing newline, writes speaker attribution when output joins context, and clears live output only after entry placement. These behaviours need to survive a job rewrite. +- In-memory shell jobs disappearing after a worker crash match the previous process-local behaviour. `persist: true` can instead report `lost`; it cannot discover whether an unkeyed external command ran before the crash. Do not silently retry non-idempotent effects. Status: job sandwich/crash semantics were already decided in target-design.md. +- Keeping a running job alive and preventing relocation are distinct from counting viewers. A job should retain its owner until it settles; ending the type or explicitly stopping work may abort it. Status: proposed job-lifetime rule, not a known owner decision. + +### Drafts and storage + +- If drafts remain keyed by conversation and author, a readable/watch for one keyed document is missing from the original API. The storage eviction wording referring to documents that are "not watched" suggests such watching, but does not define it. A singleton live map is an alternative already allowed by the design. Status: open API question; the later per-conversation draft document is a recommendation. +- The next numeric draft id and additions must be committed together. A singleton `{ next, byKey }` is a small way to avoid a second counter transaction. Claims must choose and remove items within the transaction, not decide from a stale outside read. +- Long-lived in-memory keyed drafts need explicit retirement. Persisted unclaimed drafts for guests who never return need a retention policy too. No automatic garbage collection based only on reference disappearance was proposed. +- Tags for data inside a worker's per-session document store may be redundant for session deletion, because deleting that store removes all of it. Server-side per-session docs still need tagging or owner cleanup. Status: cleanup design remains open. +- A recorded restore intent before aborting a Durable submission is a proposed recovery protocol, not proof of cross-store atomicity. Recovery must distinguish an input that was actually withdrawn from one that raced into the transcript, or it can restore duplicate text. Status: open; shared Durable docs were explicitly left open by the owner. + +### UI reload and cross-plugin interaction + +- Replacing the web `Main` component identity remounts its subtree. Composer state and retained history survive only because their providers/caches are in host code. Local scroll/focus and artifact iframe execution state can still reset. An unrelated wrapper change can destroy a user's running HTML preview. Status: UI renderer reload versus leaving is explicitly open. +- `TuiRuntime` already keeps drafts, submissions and anchors outside `ChatSurface`; attachments, reply targets and in-flight view flags are exceptions. A proposed `Composer` is another way to move them outside replaceable renderers, not a claim that p3 must use that exact object or API. +- Closing the entire TUI overlay stack on every `TuiCommands` table change affects unrelated `/reply`, `/fork`, `/downloads` and `/artifacts` overlays. Track which entry opened each overlay and close only ones whose implementation changed. Status: open UI lifecycle recommendation. +- A `ChatSurfaceKey` typed surface was the old loader-D fix for cross-plugin `instanceof`. The ownership review instead proposed a downloads list API and `Chat.canReply` so those consumers need not reach into the surface at all. Status: avoiding cross-plugin class identity was already decided; the exact replacement is proposed. +- For reply support, alternatives considered included a client `ComposerFeatures` behaviour registry and a send-router contribution declaring reply support. In both alternatives, chat-mode should not mutate chat's private surface or composer flags directly. Status: proposed extension-point shape. +- A web activation failure currently replaces the view with a retry screen, and Retry uses `WorkerControl.retry()` to restart the worker. Retrying just the failed object reference may be sufficient under object backoff/retry semantics. Restarting the whole worker is a potential unnecessary disruption. Status: open low-priority recommendation. + +### Ownership and core boundaries + +- The shared conversation must not be instantiated separately for each viewer's principal if that would duplicate its owned data. The proposal was a stable shared local owner plus per-principal remote faces that authorize and filter; alternatively the contract system needs an explicit shared-data identity. Status: proposed boundary, not an owner decision. +- Rules on conversation data belong at its owner, including read-only and cwd-following policy. An authorized remote face may stamp the principal, but other entry points, including peer input, must not bypass the owner's checks. Status: ownership and creation-time policy are owner decisions; exact face/owner layering remains a recommendation. +- A session's catalog model should be a reported mirror of the root conversation's actual model, not a second independent authority updated only by one UI path. Session should report the change so plugin-originated changes also update the catalog. Status: proposed reconciliation. +- A server broker should check current worker generation through the worker-process owner, not inspect a private workers map. Likewise archive status is obtained through session/catalog operations and rechecked after startup. Status: ownership principle decided, exact methods proposed. +- Cross-session input should use structured core-owned author data. Parsing a model-facing text prefix leaks the input format into rendering and authorization code. A guest message must not inherit an earlier owner's authorization, and nested calls must inherit the initiating turn rather than the latest racing input. Status: proposed format; current security behaviour must be preserved. +- `originOf` doing a Durable commit-time scan is a boundary case: a persisted tool phase needs data from the transcript even though p3 must not hand Durable Chord v2 objects. A core-owned plain-data format or an execution API offering the appropriate read keeps that boundary narrow. Status: open shared-Durable-document/API question. +- A full Durable harness cannot simply be passed as a host value to every plugin under the new ownership principle. The first report's host-value sketch is superseded. Plugins need owner methods or narrowly scoped local callbacks returning data; Durable receives its own plain types. Status: owner decided no cross-plugin natives and p3/Durable type separation. +- Model metadata can be returned as plain data by a model owner, but the harness needs streaming and synchronous model-access APIs as well. Replacing `HarnessConfig.models()` with only `model(ref)` does not yet solve the whole integration. A worker-side adapter around owner methods or a stable facade is still needed. Status: open interaction with models, not resolved by the object table. +- The catalog, session and conversation proposals must not create a core god object. Session owns its conversation set and location; each conversation enforces its own content policy. Small operation-specific methods are preferable to exposing the harness and bypassing the owner. Status: owner explicitly warned against a core god object. + +### Blobs, copy/delete and native placement + +- Keeping file bytes under one server owner avoids starting a worker for HTTP downloads, artifact lists or previews. Workers can upload snapshot chunks using the owner API. An index by conversation also removes the TUI downloads dependency on the loaded transcript. Status: recommendation, with transfer trade-off still open. +- Conversely, putting blob ownership on the worker makes HTTP reads depend on worker startup and transporting the bytes back to the server. The owner location must account for both producers and readers, not simply follow the current plugin location. +- The server-owned artifact proposal moved the per-conversation index out of Durable because it is independent of compaction/rewind. Preserve its special fork semantics explicitly: forks start with an empty index, while a whole-session clone copies retained files/index data. Status: proposed design; the ownership decision alone does not settle persistence semantics. +- Saving an artifact file and index through one owner narrows the protocol but does not automatically make filesystem publication and document commit atomic. The earlier discussion noted the crash after `Artifacts.save` and before `api.commit` (`artifacts/tool.ts:44-53`); publication/recovery still needs a deliberate protocol. Status: open file-format/storage transaction issue. +- A `SessionData` registry should be an extension point invoking owners' copy/delete operations, not a way for core to write into plugin-private directories. This distinction follows the ownership principle. Status: SessionDeleted remains explicitly open. +- Artifacts/Downloads wrappers holding only a directory path are not expensive long-lived natives. File handles opened inside a request can be closed in `finally`; a path-only class need not turn the whole owner into a native object. Status: implementation recommendation; classification of transient operations must remain consistent with the factory principle. +- Splitting native objects inside the same reloaded plugin is insufficient: the new design restarts every native object of that plugin. A watch/feed that must survive chat reloads must live in a separately loaded core/plugin boundary. Status: splitting expensive natives into separate plugins is owner-decided; this is its application to chat. +- Environment identity changes on relocation rather than swapping a native value. Existing connections for another cwd may be retained only as their own owners. The old environment can end once references and fenced work are gone. Status: proposed address/lifetime arrangement under the owner's immutable-native rule. + +### Checks that did not become new findings + +- Existing peer-send dedupe uses source session plus the Durable task/call request id. The broker can retry once after a recipient worker disappears without duplicating an admitted send. Registry snapshots pinned per Durable phase preserve the old tool implementation through a reload. No additional problem was found in that trace. +- Registry versions/adapters remain necessary for tools, routes, renderers, commands and send routers. Under the first design, `SendRouters` and `TranscriptEntries` needed an explicit version. Their behaviour remains an extension point, not a policy registry for someone else's data. Status: versioned immutable registries already decided. +- The original `viewer` resource should disappear in favour of reference-based presence, filtering lingering holds for viewer counts but still counting them for worker keep-alive. Status: reference-based direction already in target-design.md; link-drop/linger/count-per-origin details remain open in redesign-decisions.md. +- Routine serialization prevents old and new handlers both reconciling the same pending action during reload. Commit-wait loops should exit when their signal aborts. Status: serialized routines were already decided; overlap/latest policy remains open in redesign-decisions.md. +- No additional issue was found with transcript entry display registrations shared between web and TUI, server route authorization during an already running request, or attributed shell input when model context is included. +- The initial migration estimate assumed porting p2 in place. The owner later explicitly chose isolated p3 with no migration. The subsequent object list was itself a proposal, not a requirement to preserve p2's plugin layout; the functionality-map step was requested to rethink those boundaries from behaviour instead. diff --git a/apps/p2/docs/p3-reviews/plan-review4-summary.md b/apps/p2/docs/p3-reviews/plan-review4-summary.md new file mode 100644 index 000000000..69048db1a --- /dev/null +++ b/apps/p2/docs/p3-reviews/plan-review4-summary.md @@ -0,0 +1,21 @@ +# Review round 4 (Fable, Astra): the full plan as of `c991bdf02`, summary for the owner + +Both reviewed every decision of 2026-10-21 as one design. Verdict: keep one public `WatchHandle` and one small +synchronous publication mechanism; drop the `replicatedState` bridge; defer IndexedDB. Nothing below is decided. + +| # | Sev. | Finding | Proposed fix | Source | +| --- | --- | --- | --- | --- | +| W1 (decided 2026-10-21: adopt and mark in the step, one synchronous flush after) | blocker | "Derives and the wire encoder run inside the commit step" is unbuildable: the step is short and shared by every owner; a throwing derive after the backend commit cannot roll back; 30 encodes block every owner. Moving only Durable's adoption leaves its views behind (published later, `session.ts:765-818`); flushing each listener separately tears (`view` new, `graph` old). Durable's views scan every mounted conversation per commit | The step only **adopts and marks** (pointer swap, dependents dirty, ops appended by reference to each hold's batch and each watch's queue); a **flush** right after the step, before any listener or await, recomputes dirty derives in dependency order, encodes one batch per link and schedules watches; a throwing derive fails only its member (`broken`); changes dispatched once per conversation/document id; the same publication boundary for in-memory commits, registry changes and inbound wire batches (no store seq); Durable's `advance` called synchronously (no microtask), ideally inside `tx.adopt`; a `publishing` flag makes reentrant `self.tx`/`watch`/`release` throw | Fable 1, 2; Astra 1, 7 | +| W2 (decided 2026-10-21: explicit retirement in Chord and Durable; routines.on runs with watch.value first) | high | Durable's watch is not generic: retirement inferred from `value === null` (`observation.ts:228,266`); `start()` never delivers the acquisition value | explicit retirement event; `routines.on` runs once with `watch.value`, then drains | Astra 2 | +| W3 (decided 2026-10-21) | medium | a stopped watch keeps its listener, detach/report/replace closures and cancellation refs (`observation.ts:279`): a retained stopped handle pins the Session and plugin closures | clear on termination; GC tests count retained revisions per watch, also for stopped-but-referenced handles | Astra 5; Fable 3 | +| W4 (decided 2026-10-21: two kinds, Watchable<Json> with ops and NativeWatchable) | major | `Watchable<T extends Json>` conflicts with `self.writable` and composed registries (functions, natives) | `Watchable<T>` unbounded; exposable members are `Watchable<Json>` (Q14's check); native watchables send `[["r", v]]` (owner's proposal) | Fable 5; Astra 6 | +| W5 | major | design §24, Q14 and §12 still name `replicatedState` and `cell.ts` | text fix with the decided model | Fable 4 | +| W6 (decided 2026-10-21: connect, load, resume) | major | worker start: `load` (barrier) before `connect`; a host-lifetime store with `self.get(Catalog, {})` (a server type) parks 60 s | connect first, then load; or forbid remote `self.get` in host-lifetime factories (use `ctx.get` in routines) | Fable 2 | +| W7 (decided 2026-10-21: automatic, host kind and type only) | major | P3 address must never contain the session id, or a split or imported copy orphans every Chord document | owner strings carry the host kind and type, never an instance | Fable 3 | +| W8 (decided 2026-10-21: every str() column as UTF-16 BLOB; revisit note: Armin) | high | flat columns make unindexed text queryable, but only indexed text is UTF-16 BLOB: orders differ, lone surrogates lost, adding an index changes results; `:to \|\| substr(...)` yields TEXT; BLOB offsets must be code-unit aligned | one lossless representation for every queryable text column; casts and alignment in the backend spec and conformance suite | Astra 3; Fable 8 | +| W9 (decided 2026-10-21: no compaction at first) | high | "keep the last write per row" compaction loses data with `update` and bulk operations | no compaction at first: ordered operation batches | Astra 4 | +| W10 (decided 2026-10-21: read_after_bulk) | major | `tx.get` overlay over pending `updateWhere`/`deleteWhere` needs a JS evaluator on every backend | after a bulk op in a transaction, `tx.get` on that table throws `read_after_bulk` | Fable 6 | +| W11 (decided 2026-10-21: per-column JSON versions, row v for eager steps only) | major | migrations: a `rows` step ships one record per row (1M rows → 1M records); resuming needs durable progress | record a step as one operation when expressible in `set`; per-row only for pure JS transforms; persist step and last key atomically with each chunk | Fable 7; Astra 8 | +| W12 (decided 2026-10-21) | minor | in-memory (P2) commits have no publication id or order | the owner's next publication id, never a store seq, never the commit queue | Fable 9 | +| W13 (decided 2026-10-21) | minor | `mintId` over five tables; who flushes a link's batch; `count` is an aggregate | state: max over all tables at open; the flush unit (W1); keep `count` | Fable 10, 11 | +| W14 (decided 2026-10-21: a, b, d cut; c kept; e done) | cut | IndexedDB backend; the `replicatedState` family as public API (Chord and Durable); `stopped` and `cancelled` as one reason; derive `previous`; per-row `rows` as the default | | Fable 4; Astra keep/cut | diff --git a/apps/p2/docs/p3-reviews/rel-sol.md b/apps/p2/docs/p3-reviews/rel-sol.md new file mode 100644 index 000000000..14d3237ad --- /dev/null +++ b/apps/p2/docs/p3-reviews/rel-sol.md @@ -0,0 +1,36 @@ +# rel-sol — focused adversarial reload and rollout review + +Scope: `target-design.md` §15 and `plugin-loading.md` §§4.1, 7.4, 7.5, on pi-private `6559589bd`; read-only review against real p2 plugins and runtime. Assumes §§7–14 as given. This records review conclusions and recommendations, not implemented behavior. References to an "owner decision" below mean the decision is stated as decided in the target design; the proposed fixes and open questions are **not** known to have been decided by the owner. + +## Verdict + +**§15 is not sound enough to build as written.** The clean rollout path is plausible, but the start barrier, registry admission, removal, and overlapping-install rules need tighter definitions. These are design findings, not claims about the current loader. + +## Findings (ordered by severity) + +1. **Durable work can remain blocked after removal or a failed first load.** A subagent reporter can commit its `report` checkpoint, then `subagents` is removed and the worker restarts. The next phase has no `p2.subagent-reporter` definition, so its answer is never delivered. A pinned snapshot protects a phase already running, not one resumed after a crash. The start barrier also permits an “optional” facet to fail before Durable resumes. See target-design.md §15 (lines 446–452), `plugins/subagents/tools.ts:118–164`, and `packages/durable/src/harness/scheduler.ts:1229,1366–1381`. **Fix:** treat definitions needed by unsettled Durable tasks as required; block removal or retain their definitions until the tasks settle, and check this before `resume()`. **Status:** Open. The decision to allow optional-facet failure and to remove types in place is stated in §15; this necessary Durable-specific exception is not resolved there. + +2. **The start barrier can deadlock against its own worker admission.** The subagents host-lifetime contributor holds the server’s `Subagents` object (`plugins/subagents/worker.ts:35–39`). Today the worker admits its IPC link with `control.ready()` *before* awaiting `host.ready()` (`src/hosts/worker.ts:98–105`). If “wait until every contributor is live” and “route nothing until it reports” also delay that link admission, the hold cannot complete, so the worker cannot report. This applies to a worker spawned mid-rollout too. See target-design.md §§14–15 (lines 417–418, 446–457). **Fix:** specify an early transport/control admission distinct from application routing; gate external calls, not the link needed for bootstrap holds. **Status:** Open. The §14 start barrier and §15 install-before-routing policy are stated design decisions; their ordering needs to be reconciled. + +3. **Publishing registry entries before their resources are ready bypasses the object gate.** Radius registers a request resolver that immediately reads `connection.value` (`plugins/radius/server.ts:46–48`); HTTP invokes resolvers directly from the registry (`src/hosts/server.ts:493–500`). Between publish and resource replacement/open, a request can invoke a new resolver against an unready resource. Retaining the old resolver is not necessarily safe either if its resource has been replaced and its handle is stale. See target-design.md §15 (lines 434–440). **Fix:** make registry invocation readiness-aware, or keep the old contributor *and its usable resources* until the replacement is ready, then switch exposure together. **Status:** Open. The publish/after division and no-entry-gap goal are stated decisions; direct registry calls need a rule that actually enforces them. + +4. **A rejected facet can break a same-host registry contract without a binding failure.** For example, change chat’s `SendRouters` interface while changing `chat-mode`; if `chat-mode` setup throws, its old router remains, while new chat can read and invoke it directly (`plugins/chat/worker.ts:588–596`; `plugins/chat-mode/worker.ts:131–169`). The object-type bind rule does not protect that function call. Section 13 describes adapting old entries, but §15 does not say that an incompatible retained entry refuses the *new registry owner’s* publication. See target-design.md §§13, 15 (lines 395–401, 429–438). **Fix:** validate the complete proposed table, including retained entries, against its owner version; reject the dependent publication unless an adapter covers each old entry. Report the resulting mixed-version state, not just the facet that threw. **Status:** Open. Per-facet setup rejection and mixed versions within one host are expressly accepted in §15, but the claim that the same contracts protect every cross-plugin interaction needs this validation, including old registry readers as well as old entries. + +5. **Removal contradicts the promise that running work finishes.** A chat shell is launched after its method returns; ending its object closes `shells` and aborts the command (`plugins/chat/worker.ts:185–195, 320–347`). If chat is removed while it runs, the command may already have changed files, while its subsequent state write or transcript entry can fail (`:270–315`). A retry can repeat the external effect without the original result being recorded. See target-design.md §15 (lines 449–452). **Fix:** either refuse removal while such work is active, or define removal as cancellation with a durable, visible terminal result. Do not apply the ordinary “running work finishes” claim to removal. **Status:** Open. In-place removal and non-draining reload are stated decisions, but their interaction for already-running work is unspecified. + +6. **Successive installs need explicit per-host serialization.** Nothing in §15 or plugin-loading.md §7.5 orders R2 against an R1 whose asynchronous *after* phase is still reopening resources. R2 can publish and open its gate; an R1 completion can then replace a resource or report after R2, leaving the host’s advertised revision different from its live code. A server restart during a rollout makes late reports particularly important to discard. See target-design.md §15 (lines 429–440, 455–467); plugin-loading.md §7.5. **Fix:** serialize publish/after per host, retain only the latest desired complete code set, and fence every completion and report by revision *and incarnation*. **Status:** Open. Complete-code-set revisions with server incarnation are stated decisions; install ordering and stale-completion handling are not. + +7. **`installed` at an older revision has no mismatch rule.** The stated cases cover a provider ahead, behind *and installing*, rejected, or at the same revision (`target-design.md:468–476`). In rapid rollouts, a client at R2 can meet a worker still reporting `installed` R1 before its R2 install starts. This is neither “behind and installing” nor final; parking without a known target relies only on the deadline. **Fix:** responses must include the provider’s desired/pending revision as well as its installed revision. Define a behind-but-not-installing state: initiate reconciliation if reachable, otherwise fail or time out with a specific reason. **Status:** Open. Parked calls, mismatch status and deadlines are stated decisions; this state is missing from the comparison table. + +8. **The readiness gate opens before eager singleton migration.** Section 15 orders “then the readiness gate opens” before migrating exposed singleton documents (`target-design.md:439–444`). A newly admitted call or subscription can therefore reach a v2 member while its stored value is still v1, despite the stated rule that v2 operations never run on v1 state. **Fix:** migrate eager singletons before opening that object’s gate; if one migration fails, expose its specified per-member error when the gate opens. **Status:** Open. Lazy migration of unloaded/keyed documents and per-member failure for exposed singletons are stated decisions; the eager migration/gate order needs correction. + +## Additional review points, open questions and boundary cases + +- **Clean reconnects, new workers and late clients.** The revision/incarnation scheme and install-before-routing rule address ordinary reconnects and late clients **if** enforced for every new worker, web page and TUI host. Current `src/runtime/server.ts:380–389` chooses a worker's init plugins at spawn, and `src/hosts/web/main.tsx` and `src/hosts/tui-client.ts` use their own client connection/bundle lifecycle. A worker spawned during rollout must be tagged with the complete desired revision, admit its bootstrap link as in finding 2, reconcile it, and withhold application routing until it has reported. A client that was disconnected, hidden, or installed late must reconcile the current complete set on reconnect; a server restart must force reinstallation across incomparable incarnations. **Status:** Intended by §15 (owner-design decision), but the admission/progress ordering in findings 2, 6 and 7 remains open. No separate defect is claimed for an ordinary successful reconnect. +- **Partial rollout and rejected facets.** A server *load* error is specified to stop the rollout before publishing; a setup rejection *after* load can leave that server on mixed facet versions while workers and clients advance. Likewise a worker rejecting a facet is not necessarily a reason to stop the whole rollout. These are deliberate no-cross-host-transaction choices. The provider's reported revision must not pretend to name the complete code set when it retained an old facet, and the rejection status and plugin-specific reason must travel to affected bindings. Validate retained registry entries, shared document assumptions and old readers before treating this as contract-safe (finding 4). **Status:** Mixed versions/no votes decided in §15; safe detection/reporting of same-host incompatibility open. +- **Project installs and base rollouts.** Project revisions install per worker, outside the base rollout (`plugin-loading.md` §7.4), while the worker also receives base facets from the server (`§4.1`, `src/runtime/server.ts:380–389`). An overlapping project install and base R1/R2 install needs a single per-host publication order and an explicit representation of both desired sets; comparing only the base revision must not mistake a project facet's state for a fully reconciled worker. **Status:** Separate project revision is stated in §7.4; combined ordering/identity is open. No concrete project-plugin failure was established in this review. +- **Routines and parked calls.** §11 allows a routine's old handler to finish, with a deadline before starting its replacement. A worker routine waiting on a server call that is parked for the worker's pending install can delay that install itself. The deadline makes the wait finite, but an old handler that later resumes may duplicate a new routine's reconciliation unless the operation has its own idempotency key. The subagents reporter uses submission request IDs (`plugins/subagents/tools.ts:127–133,155–163`), an example of the required protection. **Status:** The routine deadline and reconciliation rule are stated decisions; ensure the install path does not wait on a handler blocked on that same install, and document per-operation deduplication where appropriate. This is a review question, not a separately established finding. +- **Resources and old code.** §12 expressly allows a replaced native to error for old running work and says old handles become stale. This limits §15's broad “running calls and jobs finish with their old code” language: they finish execution under old code, but need not succeed or preserve an external operation after resource replacement. Code mode's running scripts depend on a sandbox resource (`plugins/codemode/worker.ts:22–40`); removing code mode has an explicit cancellation implementation (`plugins/codemode/calls.ts`). **Status:** Resource replacement semantics decided in §12. Open recommendation: state the limitation and require a terminal result or a refusal for effects that cannot safely be interrupted, especially removal (finding 5); do not promise uninterrupted work solely from code pinning. +- **Preparation versus publication.** Explicit reload, never on save; no factory trial, migration dry run or live-consumer check at prepare; install means code published rather than every object live; per-object failed/backoff/retry are stated owner-design decisions (§15). These are not themselves defects in this review. They make clear post-publish failure states, per-member migration errors, validated registry tables and readiness gates necessary rather than optional preflight complexity. **Status:** Decisions stated in §15; concrete gaps above remain open. + +Read-only review: no implementation changes or tests were made. diff --git a/apps/p2/docs/p3-reviews/rev-ho-fable.md b/apps/p2/docs/p3-reviews/rev-ho-fable.md new file mode 100644 index 000000000..45ce0f91c --- /dev/null +++ b/apps/p2/docs/p3-reviews/rev-ho-fable.md @@ -0,0 +1,696 @@ +# rev-ho-fable — adversarial prototype and target-design reviewer + +This file records the findings, review points, open questions, and recommendations I produced during the Chord v2 / p3 design reviews. It intentionally includes superseded designs and prototype-specific findings so the reasoning is not lost. + +Status labels: + +- **[DECIDED / SUPERSEDED]**: the owner subsequently chose a different design or explicitly addressed the point. +- **[ADDRESSED]**: a later prototype/design revision addressed the specific issue, though the prototype was not merged. +- **[OPEN]**: no later decision in this conversation clearly resolved it. +- **[KEEP]**: implementation property or test coverage worth preserving. + +--- + +## 1. Drain/handover prototype, first adversarial review + +Branch reviewed: `chord-handover`, initial local handover model. + +### 1.1 Re-entry after one closure member drained — blocker + +**[ADDRESSED in the prototype; drain design later SUPERSEDED]** + +`object.ts:1667` admitted any root with work in *any* handover-group member. Members handed over concurrently, so A's steps (migrate, close exclusive, open) could run while B still drained. B's old call could then call A and be admitted to A's old implementation mid-steps. + +Concrete trace: A and B both bumped. A drained instantly and began `close conn1`. B's old call did `ctx.get(A).write`; A v1 ran after its resource was revoked, `conn.value` threw `stale`, and `count.change` wrote `{n:99}` to the old cell. The staged `{m:1}` was then installed and the write silently disappeared. + +Recommendation: drain the whole closure before any member's steps, or stop admitting re-entry to a member once its drain finishes. The prototype adopted a closure-wide quiet point and re-entry only until then. + +### 1.2 A second effect swap during pending asynchronous cleanup — major + +**[ADDRESSED; serialized routines retained in current target]** + +A second `swap()` could find no active run and start immediately while the first asynchronous cleanup still ran. The first completion then bailed because its swap generation was stale, leaving `stalled` permanently true. New and old setup effects could overlap. + +Trace: v1 setup had a stubborn asynchronous cleanup; reload v2 reported pending; reload v3 started v3 while v1 cleanup remained; after v1 settled, `stalled` remained true. + +Recommendation: keep a per-slot `#stopping` promise, chain subsequent swaps on it, and clear stalled state when that cleanup settles regardless of later swaps. This became the current rule that routines are serialized per routine on reload. + +### 1.3 Derivations installed from prepare-time values — major + +**[ADDRESSED; trials later SUPERSEDED]** + +The prototype precomputed derivations during prepare and installed that value at publication. Old code could write source cells while prepare waited on another object. + +Trace: x v2 derived `n*2+1000`; while y's `init` blocked prepare, x v1 changed `n` from 1 to 5. Publication installed 1002 instead of 1010 and it stayed stale until another write. + +Recommendation: recompute synchronously at publication before mutation, or attach without the prepare-time value. + +### 1.4 New effect handler received old-shaped buffered state — major + +**[ADDRESSED; state is now transactional documents]** + +State-source events buffered across a handover were delivered to the new handler before the migrated reset. + +Trace: observations were `v1:{n:1}`, then `v2:{n:13}`, then `v2:{m:13}`. New code acted on v1 data. + +Recommendation: queued values from readable state are not events; drop them on swap and resubscribe for the current snapshot. Only real event sources buffer data. + +### 1.5 An operation started by old code after publication got a new stamp — major + +**[ADDRESSED; operations replaced by jobs]** + +A pre-publication call could call `job.start` after publication. The operation was stamped with the current publication even though it ran old code, so old resources closed before the operation settled. + +Trace: old call started v1 operation after compatible v2 publish; old `conn1` closed while the operation ran; it later failed `stale` instead of returning 1. + +Recommendation: stamp detached work with the publication of the declaration/code generation that created it, not the wall-clock publication at `start`. + +### 1.6 Failed handover could not revert to last-live code — major + +**[ADDRESSED; drain transitions later SUPERSEDED]** + +A migration staged v2, a later exclusive open failed, then reloading v1 was rejected as version lowering from staged v2 even though the last-live v1 value remained intact. + +Recommendation: plans start from the last-live version; reverting to that version discards staged transition state. + +### 1.7 Peer-held references blocked breaking rollout — major + +**[ADDRESSED; current rollout uses revision/status mismatch handling]** + +Connection-owned v1 references were treated as local live consumers. A server provider could not bump to v2 while any worker held v1, so the rollout never began even though that worker was meant to install v2 in the same rollout. + +Recommendation: peer holds do not refuse local publication. End/reacquire them with a retryable `contract_pending`/revision state. Current design instead fails/waits at bind by revision. + +### 1.8 Removal accepted while changed code still declared the removed dependency — minor + +**[ADDRESSED in prototype; removal semantics remain an OPEN consistency point below]** + +Removal checks skipped changed types and ran before their trial factories. A changed holder could install with a now-gone `self.get(B)` reference. + +Recommendation: check prospective declarations after factories run, or refuse lookup of a type removed from the prospective service table. + +### 1.9 Handover wait deadline ignored drain deadline — minor + +**[ADDRESSED; drain design SUPERSEDED]** + +Waiting calls used `readyTimeout` only, although a reload could specify a longer drain plus grace. Calls timed out while the handover was still within its own deadline. + +Recommendation: fence wait budget must cover drain, step, activation, and grace budgets. + +### 1.10 Reconnecting children received redundant full reloads — minor + +**[ADDRESSED; current target uses complete-set revisions]** + +Every reconnect reinstalled named child modules, restarting effects even when the child already ran them. A connection open before hello could also receive both rollout install and hello install. + +Recommendation: carry an artifact revision in hello/install and skip equal revisions. + +### 1.11 First-round positive observations + +- Publication counter ownership and `releaseAfterOwners` were coherent for ordinary old calls and effect work. +- `linkedSignal` detached listeners correctly, avoiding generation leaks. +- Whole-suite generation GC test was meaningful: it tracked closure sentinels with `FinalizationRegistry`. +- Re-entry across links preserved root ids correctly in the prototype. +- Retry progress avoided double migration/open/close in covered paths once staged-state logic was corrected. + +--- + +## 2. Drain/handover prototype, second review and facet object + +The first-round tests were rerun after fixes. Findings 1–7 were confirmed fixed; removal was correctly refused; wait budgets and revision skipping were fixed. + +### 2.1 Facet-object activation failure was silent and produced an orphan repair — major + +**[Prototype-specific; OPEN as a startup/error-handling test requirement]** + +The facet hold was constructed before `object.start()`, so `RefImpl.ready` captured the initial resolved readiness. `install()` resolved while the facet object was still activating; a failed resource open was reported nowhere. Failure forgot the object from `host.objects` but retained it in `facetObjects`. A fixed reload could make that orphan `live` outside the host tables: registrations stayed absent and `host.close()` did not close its resource. + +Recommendation: start before creating readiness-dependent holds, and either retain failed host/facet objects as failed live identities or drop and recreate them consistently. Add startup-failure/retry/close tests. + +### 2.2 Failed host-lifetime object was never pinned again — major + +**[Prototype-specific; OPEN as retry semantics]** + +A gone bootstrap ref remained in `hostObjects`, causing reload preparation and repinning logic to treat it as present. Later external `get` could create a transient object, but its host registration disappeared when released. + +Recommendation: a gone bootstrap ref is absent; reload must reacquire and pin it. + +### 2.3 Prepare deadlocked against an old-code activation — major + +**[DECIDED: factory trials removed]** + +Old activation waited for `whenReloadDecided`; the trial's new factory held that same object and waited for its readiness. Reload timed out. + +Trace: Y v1 activation blocked in old open; X v2 trial declared `self.get(Y)`; Y waited for reload decision, reload waited for Y ready. + +Recommendation at the time: do not await changed old activations in trial hydration, or let them restart at publish. Final design removed factory trials entirely. + +### 2.4 Old served-object call creating a sibling during facet handover deadlocked — major + +**[DECIDED: drain/closure handover rejected]** + +A draining old call did `ctx.get` for a new sibling served by the same facet. Sibling activation waited for the facet's handover fence; the fence waited for the old call. It ended only at the drain deadline. + +Recommendation at the time: activate under old service and join the closure, or reject immediately rather than deadline-stall. Transactional reload removes this closure mechanism. + +### 2.5 Registry entry called without ctx deadlocked the closure — major + +**[DECIDED: no late-bound registries; immutable tables]** + +A contributor called a registry helper with a synchronous signature and no `ctx`. The wrapper created a fresh root with a never-aborting signal, then waited on its own contributor's fence. Production default wait approached 72 seconds. + +Recommendation at the time: no-ctx calls either use old code during drain or fail at the drain deadline. Current design uses immutable old registry tables rather than late-bound wrappers. + +### 2.6 A synchronous late-bound entry became a Promise only during reload — major + +**[DECIDED: no late binding]** + +`defineRegistry<() => string>` returned a plain string normally, but while admission waited it returned an async IIFE/Promise. Runtime behavior violated its type only during reload. + +Recommendation: entries must be consistently async, or synchronously fail while unavailable. Current immutable-table design removes the wrapper. + +### 2.7 Stalled served object blocked every future reload of its facet with no cancel API — major + +**[DECIDED: no drain/stalled model]** + +One signal-ignoring call left every closure member updating forever. A repair reload was refused `handover_running`; host API had no way to force-dispose host-principal objects despite the design calling stalled work cancellable. + +Recommendation at the time: explicit `host.cancel(address)`/force disposal, or repair reload may force-dispose stalled referenced members. Current no-drain design eliminates this state. + +### 2.8 Facet setup could not hold a type from a later facet in the same load — minor + +**[OPEN: still a requirement in plugin-loading]** + +`host.load` claimed load order independence, but facet setup was synchronously run in each `install`; `self.get` of a type served by a later facet failed `unknown_type`. + +Recommendation: two-pass load—register every facet's services first, then activate facet objects/eager objects. + +### 2.9 New bump `init` saw unmigrated sibling state — major + +**[DECIDED: cells/init model replaced by transactional documents]** + +A v2 `init` ran before staged migrated values were installed. It read another v2-declared cell and received its v1 shape. + +Trace: `settings` migrated `{n:7}`→`{m:7}`; `cache` v2 init read `settings.value.m`, but observed `{n:7}` and silently stored `undefined`. + +Recommendation: run init against a staged prospective view or disallow such dependency. Transactional documents now migrate on access and old definitions throw. + +### 2.10 Other second-round observations + +- Facet generation GC held across repeated parallel and exclusive reloads: only the current sentinel survived. +- Served objects joining during facet handover eventually activated on new code; releasing a member during handover disposed it after install. +- A finite exclusive open exceeding the step deadline was closed and made the object failed, while retry reopened it unbounded at prepare and succeeded. This contradicted “past deadline = stalled” and showed inconsistent first-attempt/retry semantics. + +--- + +## 3. State-design reviews before transactional documents + +### 3.1 Every cell has init; optional migrate review + +**[SUPERSEDED by transactional documents]** + +1. **Memory-only reset was too easy.** Once every bump had `init`, forgetting `migrate` silently reset drafts/UI state. Recommendation: require explicit `migrate` or `reset: true`. +2. **Mirror stale after resource replacement without cell bump.** `db` v1→v2 could change rows while mirror cell kept old value because init only ran on a cell bump. Recommendation: cell deps on resources, rerunning init after dependency replacement. +3. **Mirror catch-up gap.** Init read source at T1; effect subscribed at T2. External changes between them were lost. Recommendation: subscribe before catch-up or return `{value,since}` cursor. +4. **Init-only siblings had no ordering.** One init could read a sibling still uninitialized; parallel init retried forever. Recommendation: declaration-order init or explicit deps. +5. **Paged clients lacked cache invalidation.** Live window could change shape/revision while cached older pages remained old or stale after edits. Recommendation: window carries `historyRevision`; cache keys include revision and contract fingerprint. +6. **Retry/revert side effects.** Init could write into a source, later handover fail, and revert to old code that did not know the write occurred. “Idempotent” must mean stable-id upsert, and docs must state init writes are not rolled back. + +### 3.2 `init({previous, from})` review + +**[SUPERSEDED]** + +The API itself was sensible for idempotent migration into a source. However: + +- Bumped init reran on retry while added-cell init was staged/reused, creating inconsistent semantics. +- Init could call another closure member and deadlock until the step deadline; guidance needed to forbid calls into closure siblings/dependents. +- Most importantly, init ran as new code over unmigrated sibling cells (finding 2.9). + +--- + +## 4. “Reload without drain / version fences” proposal review + +**[DECIDED / SUPERSEDED by transactional document store]** + +Verdict then: simpler and right direction, but broken if applied to mutable cells and raw natives without retaining bounded ownership waits and serialized effect swaps. + +### 4.1 Exclusive close under non-cooperative work produced native errors, not `stale` — major + +The fence guarded `resource.value`, but p2 code dereferenced once and held the native. Closing sqlite/PTY/socket under work caused driver errors or partial work, not typed stale. + +Trace: Catalog `db.create` does native work, reload closes db, subsequent native statement throws `database connection is not open`/`EBADF`. + +Recommendation then: retain a bounded ownership wait before closing exclusive resources. Current target explicitly accepts native errors on resource replacement, so this tradeoff remains deliberate. + +### 4.2 Async cleanup old/new effects overlapped — major + +Old indexing/reconciliation loop could still run while new effect starts, producing duplicate writes. + +Recommendation: retain serialized per-effect swap. **[DECIDED and present as serialized routines.]** + +### 4.3 Fence missed values that escaped handles — major + +Paths: + +- native copied out before replacement; +- old subscription received v2 reset; +- detached old derived returned frozen v1 forever. + +Recommendation: fence subscriptions/derivations as handles; copied natives require ownership/grace or accepted native errors. **[Partially DECIDED: document subscriptions through old handles end stale; raw native replacement remains restart-like.]** + +### 4.4 Migrate at publish could make publish non-atomic — major + +Prepare dry-run might pass, old code changes value, real migrate throws during synchronous publish, leaving partial host mutation. + +Recommendation: compute all live migration results before mutation or move migration onto serialized document line after publish. **[DECIDED: transactional docs migrate on access/line after publish, data kept.]** + +### 4.5 `stale` thrown in timers/native callbacks could become uncaught — major + +Trace: chat shell's sampled output timer called old cell write after bump and threw from timer callback. + +Recommendation: behavior belongs only in calls/jobs/routines; errors caught/reported; old document handles stale. **[DECIDED: no bare timers/callbacks touching state; jobs/routines catch errors.]** + +### 4.6 Partial writes became routine on every schema-bumping reload — major + +Trace: Catalog native `create` succeeds then mirror cell write stales; UI retries and creates duplicate. Login provider completes then state write stales; user repeats OAuth. Chat Durable submit succeeds then cell write stales; UI retries duplicate unless requestId. + +Recommendation then: short bounded grace for ordinary calls or transactional state. **[DECIDED: transactional documents make document writes atomic; external effects require request-id dedupe/jobs.]** + +### 4.7 Durable reservations/pinned registry phases needed protection — minor + +Pinned Durable extension phase could call old entry whose resource was replaced and receive stale/native error. Recommendation: retain immutable table snapshots and map restart-like failures to task retry, not transcript tool result. **[Immutable tables DECIDED; retry mapping still OPEN.]** + +### 4.8 Init/mirror rules were not fully expressible — minor + +SQLite mirrors had no atomic external readable and could not all become derives. Cross-object reads pushed into resource open, retaining activation cycles. + +Recommendation: keep explicit source readable contract and document-backed/plugin-database cases. **[Current kinds-of-state design addresses this structurally.]** + +--- + +## 5. Full transactional-document target design review + +Verdict: the core is much simpler and closer to Durable's proven model, but several specification gaps needed resolution before building. + +### 5.1 New code window before migration/resource reopen — major + +**[Partially ADDRESSED in later §15 wording; verify implementation.]** + +At publish all new calls reached new code; migrations and resource replacement happened after publish. Without a readiness gate, new code could read v1 document shape or missing native. + +Trace: Draft v1 `text`→v2 `body`; call immediately after publish reads `.body` before migration; derive computes against old shape. Codemode sandbox closes and takes seconds to reopen; new tool call hits not-ready. + +Recommendation: in-memory docs use Durable's `valueVersion` and migrate on read; migration commit persists the base. Objects whose resources are replacing stay behind a readiness gate. Open parallel replacements before publication where coexistence is safe. + +### 5.2 Async transaction callbacks retain deadlock class — major + +**[Durable's rule decided; enforcement still OPEN.]** + +Convention “only await tx.draft” is not mechanically enforced. Any awaited call into another object while holding line A can deadlock with B waiting on A. + +Trace: A tx awaits B method; B tx awaits A read. Both lines wait forever. + +Recommendation: transaction API/type/lint/runtime rejects promises other than branded `tx.draft`; ideally preload declared docs then run a synchronous callback. At minimum dev-mode detection and clear diagnostics are required. + +### 5.3 Rollout revisions originally lacked a comparable identity — major + +**[DECIDED: revision is `(server incarnation, sequence)` over complete code set; project revisions explicitly still open.]** + +Earlier issues: project snapshot digests cannot be ordered; server restart reset counters; rejected host could be waited on forever. + +Recommendation adopted for base rollout: incarnation+sequence, status in mismatch response, reconnect reinstalls under new incarnation. Project install ordering remains an open discussion in §15. + +### 5.4 Plugin-loading and target contradicted on load order/removal — major + +**[Load order and removal still OPEN until text/implementation converge.]** + +- Load order: plugin-loading says all facets install before eager activation; facet setup-as-factory can synchronously `self.get` a later type. Recommendation: two-pass load. +- Removal: one text says declared holders refuse removal; another says references end. Recommendation: refuse removal for factory-declared holds; call/client holds end. + +### 5.5 Jobs/effects interleaving with current code — major + +**[Effect serialization DECIDED; jobs redesigned as sandwich.]** + +- Old shell job could finish after Shell result document migrated; terminal write stale, outcome lost/interrupted. +- Old/new effect handlers could both restart same environment/index work. +- Effect could start duplicate job unless idempotency key/running checked. + +Recommendations: Chord-owned job outcome, current-code finish; serialized routines; atomic keyed job start. + +### 5.6 Presence/keep-alive link loss flapping — minor + +**[OPEN]** + +Disconnect is departure. Server↔worker link blip drops every relayed hold; `host.references(Chat)` becomes zero, idle-but-viewed worker can exit, presence/login abandonment reacts, then holds reappear. + +Recommendation: bindings lost with a retryable link count as lingering during re-hold grace; explicit client release or grace expiry is departure. + +### 5.7 Persisted docs keyed by transcript ids need retirement — minor + +**[Addressed partly by `SessionTag`; implementation sweep remains.]** + +Drafts, shell dedupe, subagent children keyed by conversation can accumulate when conversations disappear but session remains. Recommendation: tag every session-tied document and retire on session delete; if individual conversations can truly be deleted independently, provide a live-id sweep/tag. + +### 5.8 Wire atomicity originally per hold, not per commit — minor + +**[OPEN unless protocol design changed.]** + +One state frame batching member updates per hold does not guarantee atomic observation across two bindings/holds touched by one commit; frames can interleave at relays. + +Recommendation: one commit frame per link carrying every touched hold batch, applied before any listeners run. + +--- + +## 6. Jobs design review and proposal + +### 6.1 Core recommendation: effect sandwich + +**[DECIDED and written into §10.]** + +A job is detached work started atomically by a method: + +1. **Intent:** `tx.start(job, args, {key})` creates/deduplicates/admission-checks on the object's line. +2. **Effect:** old-generation `run(args, ctx)` may do external work, streams Chord-owned progress, and returns a typed result. +3. **Outcome:** Chord records outcome in its own job record, then current code's `finish(outcome, tx)` writes current-schema documents and acknowledges atomically in the same commit. + +This solves a run finishing after reload migrated the plugin document: old code never writes the document; finish uses current handles. + +### 6.2 Proposed job semantics + +- Typed `defineJob(name, {version,args,progress,result,concurrency,persist,accepts})`. +- Required dedupe key for user requests where duplication matters; Chord mints globally unique run id. +- Atomic concurrency and dedupe in intent transaction. +- `running` includes executing and pending-finish records so a stuck finish holds the busy slot. +- `ctx.progress` is typed, throttled, bounded with `OutputBuffer`/`Progress`, in-memory deltas. +- Abort is a request: signal run; run can return partial successful/cancelled result; signal-ignoring run retains its slot until it ends. +- Current-code finish retries with backoff; throw leaves outcome pending. +- Version adapters convert old args/results for current finish; terminal stored outcome is not rewritten. +- Persisted jobs only on stable identities. Live record found after crash becomes `lost`; effect is never rerun automatically. +- Running/unsettled jobs count for object lifetime, worker keep-alive, and relocation fence. +- Work that must resume belongs in Durable task; request-bound work can remain a method. + +### 6.3 External systems cannot be atomic with Chord finish + +**[DECIDED in design guidance; fundamental limitation.]** + +`finish(outcome, tx)` follows Durable's rule and cannot call Durable submission/network/another object. There is no atomic transaction across Chord and Durable anyway. + +Recommendation adopted: external write happens in `run`, keyed by `ctx.jobId`; `finish` only records Chord documents. For chat shell, Durable `requestId: shell:${jobId}` prevents duplication if submit committed but process died before Chord outcome record. + +Systems without a dedupe key cannot be exactly-once. Chord can offer at-most-one attempt, not both no-loss and no-duplicates. + +### 6.4 Concrete job classification + +- **Chat `!cmd`: job.** Must outlive browser tab; needs detached execution, progress, abort, dedupe and current-code outcome. Today's code hand-rolls all of these with `host.root`, map, resource, AbortController and cell. +- **Env test: long method call.** Caller waits; short/retryable; no detached lifecycle needed. +- **Login OAuth: job + Questions.** Must outlive one tab and be visible/answerable elsewhere; progress carries auth URL/device notices; Questions handles prompts. +- **Codemode/tool/subagent work that must resume or belongs to transcript causal order: Durable task**, not Chord job. +- **Questions `ask`: long method call.** It should die with its server-side hook/flow caller. + +### 6.5 Job abort/crash traces + +- User abort signals run; run returns partial output if possible; finish records result. +- Object end aborts run. Persisted pending outcome finishes at next activation; in-memory record may be reported and lost. +- Host crash: persisted live records become `lost`; in-memory jobs vanish. Chord does not falsely claim the external effect did or did not occur. +- Reload: old run continues; current code finishes it. + +### 6.6 Honest limitations/open questions for jobs + +- Exactly-once external write requires external idempotency key. +- Non-stable object identities cannot retain job record over host crash. +- Signal-ignoring native may linger; current design retains slot until it truly ends rather than lying. +- Interactive input document migrations can stale an old subscription; Questions avoids using a plugin document. +- A blocked old outcome without an `accepts` adapter can hold concurrency forever. **Recommendation: prepare refuses a job version bump while live/pending old-version records lack coverage. [OPEN unless added.]** +- Abort example must preserve partial output: catch abort and return data, or include last progress in failed outcome. **[OPEN wording/example issue.]** +- Live/unsettled run must hold its object alive like a reference so finish has a line/resources. **[OPEN unless specified.]** + +--- + +## 7. Questions service review + +### 7.1 Core design assessment + +**[DECIDED and documented.]** + +The service is sound in its core: + +- pending entries/resolvers live in a resource, not a document; +- `open` is a source-readable/derive over that store; +- register waiter + abort listener before publishing; +- one atomic settlement removes entry before resolving; +- late/second answer fails; +- answers validated against stored question; +- authorization checked separately for view and answer; +- rejection/link loss is not denial; asker retries with same key; +- Durable tool approval memo prevents re-asking after task replay; +- notices (auth URL/device code) use job progress, not Questions; +- per-prompt abort withdraws a question. + +### 7.2 Initial issues and resulting recommendations + +1. `open` must not be a document; otherwise old `ask` removal can stale after migration and leave ghost questions. **[DECIDED.]** +2. Resource replacement must reject pending askers unless `replace(old)=>old`; asker retries. **[DECIDED.]** +3. Link rejection must retry rather than map to deny. **[DECIDED.]** +4. Scope authorization lives in factory; guest send access does not grant approval. **[DECIDED.]** +5. Notice-only prompt/withdraw needed. **[DECIDED: notices via progress, prompt-specific abort.]** + +### 7.3 Remaining Questions review points + +- A single `Questions({scope})` contract cannot naturally have both server placement for owner scope and session-host placement for session scope under static type placement. Recommendation: separate `OwnerQuestions` and `SessionQuestions`, sharing schemas/store logic, or explicitly add branching placement support. **[OPEN.]** +- Login example using only `ctx.jobId` for every prompt in one flow risks reattaching to the previous prompt if prompts overlap or settlement races. Use `${jobId}:${promptId}`. **[OPEN.]** +- `self.derive([pending.value.readable], ...)` reads a resource value during factory/trial declaration. Better: resource handle is a source resolved at activation, or facet exposes a readable member. **[OPEN implementation/API detail.]** +- Approval hook must run after argument-rewriting hooks so approval covers final arguments. **[Documented decision; preserve in registry ordering.]** + +--- + +## 8. Resources `replace` review + +### 8.1 Handover hook assessment + +**[DECIDED.]** + +Synchronous `replace(old)` is a useful narrow mechanism for shape/container changes that preserve native identity, avoiding needless socket/process restart. Logic remains in routines; old handles stale; throwing replacement fails object and closes old value. + +### 8.2 Dependent resources must be handled + +**[OPEN unless §12 updated.]** + +A dependent opened from the old provider value remains bound to it if provider uses `replace` but dependents are not reopened/replaced. + +Trace: `watch` depends on `conn`; conn v2 replacement wraps/reuses socket, but watch retains listeners/container assumptions built from v1. Later close/restart can orphan it. + +Recommendation: after provider `replace`, every dependent whose dependency incarnation/version changed is itself replaced/reopened in provider-first order, unless its own `replace` explicitly hands it over. + +### 8.3 Accepted resource-replacement tradeoff + +Current design explicitly treats raised resource version like restart for that native: calls/jobs are not waited for or signalled; work holding raw native gets native error. This is an owner decision. Preserve tests around terminal service, radius socket, sandbox and sqlite-like plugin DB replacement; errors must be caught by call/job/routine boundaries. + +--- + +## 9. Routines namespace naming + +### 9.1 `run` versus `routines` + +Initial ranking proposed `self.run.on/every/loop/custom`, with `react` and `keep` alternatives. On follow-up, I ranked **`routines` above `run`** because collisions were substantial: + +- job definition has `{run, finish}`; +- each job execution is a run; +- `custom` setup receives `run(fn)` helper; +- `self.run.custom(api => api.run(...))` gives “run” three meanings in one line. + +`self.run.every` reads well alone, but docs would repeatedly disambiguate. `self.routines.on/every/loop/custom` is longer but names standing object-owned behavior clearly (Go routines / OTP-like processes) and leaves “run” to jobs. + +**[DECIDED: latest target uses §11 Routines and `self.routines.*`.]** + +--- + +## 10. Reload without factory trials / dry-run review + +### 10.1 Overall position + +**[DECIDED: reload = restart without restarting; explicit load → publish → after.]** + +I mostly agreed with dropping the trial object universe. Trial atomicity for factory throws caused severe complexity: objects created during prepare, old activations waiting on prepare, cleanup/adoption ambiguities, and closure bugs. Restart-like failure is coherent. + +### 10.2 Keep facet setup staging + +A setup may serve types and register entries before throwing. Direct live mutation would leave a partial facet. + +Recommendation adopted: setup runs into a staging sink; throw discards that facet staging and keeps previous facet services/entries/object. Successful facet staging installs atomically with others. + +### 10.3 Registry conflict handling + +Tables must be built from successful new staging plus previous entries of rejected facets. A `conflict:"error"` discovered at publish should reject the contributing declaration/facet deterministically, not leave half a table. + +### 10.4 Publish duration + +Synchronous re-run of every live declare-only factory is acceptable and preferable to half-published objects. Measure/report one-tick duration. Derive notifications occur after adoption. Large live-object counts need benchmark/watchdog but not asynchronous publication. + +### 10.5 Start barrier distinction + +First load is not reload: Durable opens without scheduling, host values provided, all facets/eager contributors live or failed, required configuration checked, then `resume()`. Reload needs no global barrier, but per-object readiness gates and routine serialization remain. + +### 10.6 Migration error behavior + +Exposed singleton migration failure should be per member rather than killing the whole reference/object: methods and other members stay usable; member reports error; data kept. Current transactional-doc design states migration refusal on access and member error; preserve this behavior. + +--- + +## 11. Focused reload and rollout review of latest §15 + +### 11.1 Binding can wait for an install that already rejected — major + +**[OPEN unless mismatch algorithm now consults local outcome.]** + +Worker facet rejects R+1 and keeps v1; server provider is v2/R+1 installed. Worker v1 binding receives provider `{R+1, installed}`. Comparing only revision says “provider ahead, my install is coming,” so it waits to deadline, reholds and repeats. Its install already rejected. + +Recommendation: binding consults its own host's outcome for provider revision. If this host/plugin rejected it, fail immediately with that reason. Provider response alone is insufficient. + +### 11.2 After-order must cross objects — major + +**[OPEN specification detail.]** + +Resource deps cross object boundaries via references and facet closure. Terminals trace: `TerminalServiceObject.service` bumps; `Terminals.view` depends on service and its open calls service. If dependent after-step runs before provider gate opens, it uses closed/not-ready native. + +Recommendation: object's after-step waits for readiness gates of referenced provider objects and facet object; provider-first DAG, with cycle diagnosis. This is activation dependency ordering, not drain ordering. + +### 11.3 Failed re-run of an existing host object can remove required contributor — major + +**[OPEN / owner decision needed.]** + +Facet setup staging keeps old facet on setup throw, but a live host object's factory throw currently makes the object failed and removes entries. Example: models host object throws, `HarnessConfig.default` disappears; Durable resource waits in `whenRegistered` forever and worker never becomes usable. + +Recommendation: symmetry with facet setup: if a live object's re-run factory throws, keep its previous version/entries and report rejection; only first activation has no fallback. At minimum required-key wait must reject visibly after barrier rather than hang. + +### 11.4 Removal rules remain inconsistent — major + +**[OPEN.]** + +§15 says removed objects/references end. Plugin-loading says any surviving declared holder refuses removal before preparation. If removal proceeds while unchanged consumer holds removed type, its factory is not rerun; it remains live with gone ref indefinitely. + +Recommendation: one rule in both docs. Preferred: refuse while factory-declared holds exist on any host; call-scoped/client refs end. Alternative: dependency-changed consumer objects must be rerun/failed as if their plugin changed. + +### 11.5 “Registry table built once” versus old entries until live — minor + +**[OPEN wording/algorithm.]** + +Keeping an old contributor entry until its replacement gate opens implies another immutable table swap at that gate. It cannot literally be built once for the whole reload. + +Recommendation: whole table swaps at publish and whenever a contributor transitions live/failed; each table immutable and snapshots remain valid. + +### 11.6 Pinned Durable phase and raised resource version — minor + +**[OPEN p2 glue.]** + +A phase holds old immutable Extensions snapshot; resource used by old tool entry bumps/closes. Tool gets native/stale failure. That should be task retry/restart-like failure, not a transcript tool result the model reasons over. + +Recommendation: p2 tool wrapper classifies Chord `stale`/restart-like unavailable/native-close errors as retryable task invocation failure where safe. + +### 11.7 Worker spawn/install race — minor + +**[OPEN implementation requirement.]** + +Worker spawned mid-rollout may receive init list for R while install R+1 arrives before `host.load`/start barrier finishes. + +Recommendation: install queue serializes behind initial load/start barrier; init carries revision; equal queued install becomes no-op; different incarnation forces reconcile before routing. + +### 11.8 `rejected` status conflates whole-host old revision and partial facet keep — minor + +**[OPEN naming/status.]** + +A load error leaves entire host on previous revision. A setup throw keeps one facet previous while others publish. Both called `rejected`, but mismatch handling needs plugin/type granularity. + +Recommendation: `installed_partial {kept:[plugin,reason]}` or equivalent per-plugin outcome. Treat kept plugin types as rejected and successful plugins as installed. + +### 11.9 Several quick rollouts + +Per-host install requests must serialize. Every revision names complete desired set, so R+2 reconciliation can subsume skipped R+1. Parked calls belong to binding/code generation: replacement/release ends them; do not resend automatically. Durable/user writes need their own request IDs. + +### 11.10 Server restart/reconnect + +Revision `(incarnation, sequence)` is the right decision. Different incarnation is incomparable and requires complete reconcile before serving. Server must persist/derive desired complete code set independently from transient rollout state; rejected outcomes from prior incarnation cannot be blindly waited on. + +### 11.11 Late clients / clients that never install + +Background web/TUI may install late; calls park only until bounded deadline. If client code generation is replaced, parked calls end and are not replayed. A client that cannot build/install surfaces update-required/error; provider must not retain work forever. + +### 11.12 Start barrier required-key failure + +`runtime/worker.ts` currently waits for registry keys such as `HarnessConfig.default`. Required contributor failure must reject barrier with named cause; optional facets may fail and remain unavailable. Never leave `whenRegistered` pending forever after all contributors have settled. + +--- + +## 12. Plugin-loading consistency and loader/GC recommendations + +### 12.1 Explicit reload, not save + +**[DECIDED.]** + +Plugin reload is explicit (`npm run reload -- plugin`) after edits. Runtime/base changes restart affected hosts. This reduces frequency of restart-like resource failure and makes partial side effects less surprising. + +### 12.2 Complete desired code-set reconciliation + +**[DECIDED for base rollout.]** + +Every revision names all facet digests, not just a patch; each install reconciles missing/different pieces. This is necessary for skipped/reconnected hosts and rapid successive rollouts. + +### 12.3 Loader generation reclamation + +**[KEEP.]** + +Model D (base bundle + one script per facet, host packages external/passed in) is compatible with old-generation calls/jobs and immutable registry tables if nothing retains the generation. + +Preserve: + +- generation sentinel GC tests after 3×/6× reload; +- dev leak detector and heap-snapshot action; +- no behavior in resources; +- immutable old registry tables; +- compile cache disabled/bounded for in-place VM scripts; +- browser compilation-cache measurement and full reload after bounded generations if necessary. + +### 12.4 Cross-plugin identity/import guardrails + +Preserve no `instanceof` across plugin bundles, typed surfaces by name, cross-plugin imports limited to contracts/pure allowlist, host packages single instance, and contracts bundled from plugin's own versioned dependency so adapters are meaningful. + +### 12.5 Project plugin identity remains open + +Two project scopes can both define plugin `foo` with different code. Relocating a session could interpret persisted `foo.*` records/tasks with the wrong implementation. Recommendation already present: provenance includes project/plugin identity (scope digest or stable package identity), and mismatch is “plugin not available,” not execution by same-name code. + +--- + +## 13. Consolidated items to preserve from the abandoned drain prototype + +**[KEEP; several already in target.]** + +- Facet object and served-object dependency relation. +- Immutable registry table identity and snapshots. +- Per-effect/routine serialized swap. +- Publication/adoption notification queue so observers never see half commits. +- Publication/work ownership stamps where needed for deferred cleanup and generation GC. +- Revision separate from health/failure status. +- Generation-GC and repeated-reload leak tests. +- Activation retry sharing/backoff per address; explicit reload/dependency change clears it. +- Resource dependency ordering (including cross-object/facet deps). +- New activation never goes live under code already replaced. + +Do **not** preserve: closure-wide drains, re-entry/root ids, staged cell copies, transition resume progress, handover deadlines/stalled states, or late-bound registry wrappers. + +--- + +## 14. Current overall verdict + +The transactional-document design is materially simpler and more sound than the drain/handover prototype: data is on serialized object lines, old definitions cannot touch migrated documents, running calls/jobs keep their code, registry tables are immutable, and reload does not invent a second trial object universe. The remaining high-risk items are specification/implementation boundaries rather than a need for another reload architecture: + +1. binding must fail fast when the relevant install already rejected; +2. after/readiness dependency order must cross objects; +3. required contributor failure must not silently remove essential registry keys or hang the start barrier; +4. removal semantics must be identical in target-design and plugin-loading; +5. transaction await restrictions need mechanical enforcement; +6. project revision/identity and per-plugin partial-install status need a precise model; +7. jobs need coverage checks for old pending outcomes, partial-abort output semantics, and object-lifetime ownership; +8. Questions needs resolvable placement/types and prompt-specific keys; +9. wire commit batching across holds should be explicitly atomic if cross-binding consistency is promised; +10. link-loss presence should distinguish transient re-hold from true departure. + +With those resolved, §15 is sound enough to build. None requires reintroducing drain-based handover. diff --git a/apps/p2/docs/p3-reviews/rev-ho-opus.md b/apps/p2/docs/p3-reviews/rev-ho-opus.md new file mode 100644 index 000000000..1b0535a04 --- /dev/null +++ b/apps/p2/docs/p3-reviews/rev-ho-opus.md @@ -0,0 +1,838 @@ +# rev-ho-opus — adversarial reviewer of Chord v2 / p3 state, jobs, reload and rollout + +This file records the findings, review points, open questions and recommendations I produced in the review conversation. It excludes the functionality map. Status labels reflect later owner decisions or later designs known in that conversation; they are not a fresh re-review of current HEAD. + +Status legend: + +- **Decided / superseded:** the owner later chose a design that replaces the reviewed one. +- **Decided / adopted:** the later target design explicitly adopted the recommendation. +- **Confirmed fixed:** a later prototype revision and regression test addressed the finding. +- **Open:** no later owner decision or verified fix was established in the conversation. +- **Historical:** useful implementation evidence from a design that is no longer the target. + +--- + +## 1. Drain/handover prototype: first adversarial review + +**Status: Decided / superseded.** The owner later rejected drain-based handover. These findings remain useful traces of why. + +### 1.1 Blocker: a drained object could still run old code during the rest of its handover + +**Finding.** `object.ts:1667` admitted any call whose call tree had work in *any* object of the handover group. Each object's handover ran independently (`reload.ts:568`), not in dependency order. Once B had drained, a call from A, which was still draining, could enter B's old code while B migrated and swapped resources. + +**Concrete trace / test.** `/tmp/chrev/reentry.test.ts`: B drained, migrated `{n:1} -> {m:1}`, closed `conn1` and waited on `conn2`. A's call then reached B v1: its write to `n` produced `{n:11}`, its resource read threw `stale`, and B ended at `{m:1}`. The write was lost although the reload reported `installed`. + +**Recommendation.** Admit re-entry only while the whole closure is draining. Drain the closure to one common quiet point before any member runs migration/close/open steps, then execute steps in dependency order. + +**Later status: Confirmed fixed in the prototype.** The rewritten test retained the re-entrant write (`{m:11}`), and no read happened after resource close. + +### 1.2 Major: registry calls escaped contributor admission and ownership + +**Finding.** `host.ts:617` returned raw registry values. A route or tool call did not become work of the contributing object. Neither handover drain nor `releaseAfterOwners` waited for it. + +**Trace.** `/tmp/chrev/registry.test.ts`: a running route saw `close conn1`, then `route v1 ERR stale`, for both an exclusive handover and a parallel resource replacement. + +**Recommendation.** Wrap function entries so invocation enters the contributor's admission and begins stamped work with the caller's root id/context. + +**Later status: Confirmed fixed for function-valued entries; object-valued entries remained a later finding.** + +### 1.3 Major: queued `run()` closures were stamped when they started, not when admitted + +**Finding.** A closure queued before publication could begin afterward with the new publication stamp and therefore own none of its old declarations. + +**Trace.** `/tmp/chrev/queued.test.ts`: `v1 handled 1 with conn1`, then `close conn1`, then queued `v1 handled 2 ERR stale`. + +**Recommendation.** Reserve and stamp work when `run(fn)` admits the closure, or drop queued closures from a stopped generation. + +**Later status: Confirmed fixed.** The old resource closed only after the queued closure ran. + +### 1.4 Major: every failed object blocked every unrelated reload + +**Finding.** `#todo` retried failed objects in every reload. A failed exclusive open therefore rejected an unrelated facet change during prepare. + +**Trace.** `/tmp/chrev/failedblocks.test.ts`: while port 9090 remained busy, reloading an unrelated `other` facet was rejected with `EADDRINUSE`. + +**Recommendation.** Retry failed objects only when their type/plugin is in the batch, or isolate retry failure to that object and continue the reload. + +**Later status: Confirmed fixed.** + +### 1.5 Major: reverting to last-live code could not repair a failed handover + +**Finding.** Staged v2 state was treated as current, so reverting to v1 was rejected as a version decrease even though v2 had never gone live. + +**Trace.** `/tmp/chrev/revert.test.ts`: after a failed v2 port open, reload to v1 failed with `version of state count lowered from 2 to 1`. + +**Recommendation.** Plan from last-live state. A staged copy is valid only for exactly its target version/schema and should be discarded for a different target. + +**Later status: Confirmed fixed.** + +### 1.6 Major: a handover step that hung was never reported and could never be repaired + +**Finding.** Exclusive `open` and `init` after drain had no deadline. The object remained `updating`, not `stalled`; reload never resolved, later reloads got `handover_running`, and a child's serialized install queue wedged permanently. + +**Trace.** `/tmp/chrev/hang.test.ts`: an exclusive open that never resolved left `reload()` pending and the object `updating, stalled:false`. + +**Recommendation.** Give each post-drain step an aborting deadline and stalled reporting; do not let a child install queue wait forever on a post-publication handover. + +**Later status: Partly fixed for one-member closures; multi-member closure remained broken in round two.** + +### 1.7 Major: a second reload during async effect cleanup started a replacement under it and left `stalled` stuck + +**Finding.** A stopped run had already been cleared, so the second swap saw nothing to wait for and started immediately. The superseded first swap returned without clearing its stalled flag. + +**Trace.** `/tmp/chrev/doubleswap2.test.ts`: `setup v3` occurred before `cleanup done v1`; the second report said `installed`; `stalled` stayed true forever. The loop variant also showed the stuck flag. + +**Recommendation.** Keep the pending cleanup promise on the slot and chain every later start after it. Clear stalled state when the cleanup settles or a swap is superseded. + +**Later status: Confirmed fixed for loops and plain setups.** + +### 1.8 Minor: old activations delayed publication and could livelock reload + +**Finding.** Prepare waited for activating objects of changed types. A slow activation delayed reload up to `readyTimeout`; a steady stream could prevent publication forever, despite a `CodeChanged` restart path existing. + +**Recommendation.** Publish without waiting and abort/restart old-code activations synchronously at publication. + +**Later status: The direct wait was fixed; a prepare/activation dependency cycle remained a later finding.** + +### 1.9 Missing design pieces + +- Facet setup was not represented as a facet object, so objects sharing facet closures were not handed over with it. +- Dependents of a failed provider stayed live rather than failing with it. +- Handovers ran concurrently rather than in dependency order. + +**Recommendation.** Introduce the anonymous facet object, include served objects when its declarations change, propagate provider failure, and order close/open by the resource/reference dependency graph. + +**Later status: Implemented in the prototype, then superseded with the drain design itself.** + +### 1.10 What held in the first prototype + +Calls racing publication, waiting calls, stale-write refusal after install, parallel-resource ownership by stamped work, cell version checks and dry runs, no repeated migration on a plain retry, and the repeated-reload generation checks held under the tests examined. + +--- + +## 2. Drain/handover prototype: second adversarial review + +**Status: Decided / superseded.** The owner later rejected the drain design. These findings came after closure-wide drain, dependency order and the facet object were added. + +### 2.1 Major: sequential closure steps let one member freeze every later member + +**Finding.** Members' close/open steps were awaited one at a time. A member whose open ignored its signal prevented all later members from reaching their own steps and prevented the reload report from completing. + +**Traces.** + +- `/tmp/chrev/r2facet.test.ts`: served object `a` hung opening a cursor after a facet DB replacement. `reload()` never reported; served object `b` remained `updating`; later reloads were refused. +- `/tmp/chrev/r2seq.test.ts`: six independent served objects each took 60 ms, below a 100 ms step deadline. Sequential processing took 366 ms and the last two waiting calls hit `handover_timeout`. + +**Recommendation.** Execute independent members concurrently once their providers are ready. Report every member blocked behind a stalled provider. Size call waiting by the longest dependency path, not the number of members. + +### 2.2 Major: registry entry code ran with the caller's context, not work-owned child context + +**Findings.** + +- The handover deadline aborted the entry's internal work record, but not the context passed to entry code. A streaming route honoring `ctx.signal` never stopped. +- New code reached through a long-lived caller context retained the caller's pre-migration stamp, so legitimate writes were rejected as stale. + +**Traces.** + +- `/tmp/chrev/r2entryabort.test.ts`: the reload reported pending, the route kept running, and `host.close()` hung. +- `/tmp/chrev/r2entrystale.test.ts`: a new route after migration got `stale: a write from work that started before its state was migrated`. + +**Recommendation.** Replace the entry's context argument with `ctx.child(contributor, work)`, exactly as object method calls do. + +### 2.3 Major: object-valued registry entries bypassed admission and ownership + +**Finding.** Only top-level function values were wrapped. A tool object such as `{ name, execute }` was returned raw. + +**Trace.** `/tmp/chrev/r2registry.test.ts`: the old resource closed while `execute` ran; `execute` then got `stale`. + +**Recommendation.** Wrap declared callable properties of object entries, or constrain registries to function values. + +### 2.4 Major: removing a facet closed facet resources under served-object work + +**Finding.** `dropFacetObject` released its host hold immediately. The facet object disposed and closed resources before served objects completed forced disposal. + +**Trace.** `/tmp/chrev/r2facet.test.ts`: `close db1` occurred while a served call was pending; the call resumed with `not_ready`. + +**Recommendation.** Keep the facet hold until every served object has disposed; release it afterward. + +### 2.5 Major: revision skip after a rejected child install permanently stranded missing modules + +**Finding.** A child that rejected R2 could successfully install only R3's delta and then claim R3. Reconnect skipped because revisions matched, although R2's module was never installed. + +**Trace.** `/tmp/chrev/r2revision.test.ts`: module `a` failed transiently in R2, module `b` installed in R3; reconnect installed nothing and the worker had only `b` while the server desired `a,b`. + +**Recommendation.** Clear/withhold the child's revision after rejection, track revision per connection, or send the complete desired module set for every revision. + +### 2.6 Minor/major: prepare deadlocked with an old-code activation + +**Finding.** The activation waited for reload decision, while a trial/new host object waited for a reference to that activation. + +**Trace.** `/tmp/chrev/r2activation.test.ts`: reload rejected after `readyTimeout`, naming the activating target reference. + +**Recommendation.** A candidate acquisition of an old-code activating object must restart it immediately with candidate code, or prepare must not await readiness of references to changed types. + +### 2.7 Minor: no-context entry call could wait for its own handover + +**Finding.** A closure member calling a registry entry without passing a context got a new root and a never-aborting signal. It parked behind its own fence. + +**Trace.** `/tmp/chrev/r2registry.test.ts`: still waiting after two seconds; it could wait roughly the full 60-second handover timeout. + +**Recommendation.** Carry root identity through async-local context, or reject/cancel no-context entry waits at the drain deadline. + +### 2.8 Facet-generation GC held + +`/tmp/chrev/r2gc.test.ts` reloaded facet objects five times with both parallel and exclusive resources; only the current generation remained. This part held. + +--- + +## 3. State §7 design reviews before transactional documents + +### 3.1 `init({ previous, from })`: live previous value was unsafe + +**Status: Decided / superseded by transactional documents.** + +**Finding.** Passing the live last-live object let `init` mutate state outside transactions. If `init` later failed, reverting served corrupted old state with no state event. + +**Trace.** `/tmp/chrev/r2init.test.ts`: `init` drained `previous.items` then threw. Revert served `{ items: [] }` instead of `['a','b','c']`. + +**Recommendation.** Pass a structured clone or frozen value. + +### 3.2 `init` calling another closure member could deadlock + +**Status: Decided / superseded.** + +`init` ran in a fresh root after re-entry closed. Calling a sibling/self that had not completed handover parked until deadline and failed repeatedly. The author could not know closure membership statically. + +**Recommendation.** Enforce at runtime: fail fast when an init root calls an updating closure member, or permit only already-installed providers. + +### 3.3 An init deadline could be too short for data-moving I/O + +**Status: Decided / superseded.** + +Moving a large in-memory value to SQLite under the ordinary drain deadline could fail every retry. If I/O committed after abort, retry repeated it. + +**Recommendation.** Give init its own longer deadline and require one transaction plus idempotence. + +### 3.4 Mirror cell could lose or duplicate source events around init + +**Status: Decided / adopted in later design as source readables / derives.** + +**Trace.** First start: init reads m1–m4, m5 commits before subscription, so m5 is lost. Reload: an event m5 buffers while init reads through m5; replay appends m5 twice. + +**Recommendation.** Mirrors come from atomic readable sources, or use sequence-numbered events with subscribe-before-catch-up and drop events at or below the read position. + +### 3.5 Memory-only data could be silently dropped + +**Status: Decided / superseded by versioned transactional documents.** + +Because every cell had init, a bump without migrate silently reset drafts. Returning `undefined`, renaming or removing a cell had the same effect. + +**Recommendation.** Declare mirror versus memory state, require migrate for memory state, and require an explicit drop/retire declaration. + +### 3.6 Paging needed stable cursors, gap detection and cache epochs + +**Status: Partly adopted by the real Chat window/history design; general contract remained a review recommendation.** + +**Traces.** Offset paging while the window advances causes skips/duplicates. A disconnected client whose cache ends at m149 can reconnect to window m400–m449 without knowing m150–m399 are missing. Schema/source rewrites leave cached pages stale. + +**Recommendation.** Stable monotonic entry ids; window bounds `{from,to,items}`; `history({before:id,limit})`; reconnect fills the gap; cache keyed by contract fingerprint and source epoch. + +### 3.7 Cross-cell init ordering was undefined + +**Status: Decided / superseded.** + +Two bumped cells could initialize in parallel; one reading the other could observe old shape, no value or nondeterministic order. + +**Recommendation.** Forbid reading not-yet-initialized cells, support explicit dependencies, or pass staged values into init. + +--- + +## 4. Reload without drain / version-fence proposal + +**Status: Decided / rejected.** The owner replaced it with transactional documents plus per-generation handles and restart-like reload. + +### 4.1 Major: retryable stale after partial external work duplicates effects + +**Trace.** `Catalog.create` writes SQLite, then its mirror-cell update throws stale; client retries and creates another session. Search mark/index had the same source-first/mirror-second pattern. + +**Recommendation.** Do not make stale generically retryable. Require request ids for mutating methods crossing reload, or let mirror updates trigger re-read rather than fail. Provide real readable wrappers for SQLite mirrors. + +### 4.2 Major: per-cell fences miss copied values and cross-cell invariants + +Old code can copy a v1 value before publish, await, and write a derived value to an unchanged cell, Durable or another object. No changed handle is touched after publication. + +**Recommendation.** Keep a per-object write epoch: if any storage changes, old-stamped work cannot write any document of that object and its signal aborts cooperatively. Old derives over bumped documents also become stale. + +### 4.3 Major: stale-on-read breaks cleanup and timers + +**Trace.** Chat's shell timer/finally read `shell.value`. A bump could throw from a raw timer, crash the worker, skip cleanup, leave `running:true`, while the new closure no longer had the native abort controller. + +**Recommendation.** Old reads return their generation's immutable snapshot; only writes fail. Reset transient fields during migration. Abort all work owned by the object, including detached roots it started. + +### 4.4 Major: immediately closing exclusive natives is not restart-equivalent + +A restart kills old work; fenced reload left it running against closed SQLite/socket/native handles. Work in dependent objects was not necessarily signalled. Abandoned old opens could race a new exclusive open. + +**Recommendation.** Abort work of every dependent object and wait boundedly before close. New calls already wait for reopen, so this bounded exclusive-only drain cannot deadlock. Wait for abandoned exclusive opens to settle. + +### 4.5 Major: Durable pinned snapshots and stale conflict + +A phase pins an Extensions snapshot. Late binding would pair a v1 schema with v2 implementation; old binding could hit changed storage and commit a stale tool error where restart would resume the task. + +**Recommendation.** Keep Durable reservations/ownership. A pinned phase runs its snapshot code and keeps declarations alive; abort/reschedule the phase rather than commit stale. + +### 4.6 Major: effects and entries started before async activation + +**Trace.** Chat's watch routine starts while its conversation resource is reopening; source resolution throws `not_ready`, and the effect stays failed until another reload. Reset members can appear unhydrated to existing holders. + +**Recommendation.** Keep object `updating` during async activation. Start effects, entries and remote snapshots only after success; buffer events meanwhile. + +### 4.7 Minor/major: async effect cleanup overlap + +Search indexer or subagent reporter old/new loops could overlap. Old report may overwrite new state; indexing could run the same range twice. + +**Recommendation.** Replacement starts after cleanup settles for that slot; deadline/stalled is local to the routine, not the object. + +### 4.8 Minor: post-publish failure made revert impossible + +The cell migrated at publish; an exclusive open then failed. v1 code could not be restored without a version decrease. + +**Recommendation.** Retain pre-migration last-live snapshots until activation succeeds; allow revert to restore them. If reset moves memory into a source, pass previous with idempotent semantics. + +### 4.9 Verdict on version fences + +Simpler than full drain but broken at storage/work boundaries. The choices that had to change were per-cell rather than per-object fencing, stale-on-read and retryability, and immediate exclusive close. Useful pieces to preserve were ownership stamps, deferred close, Durable reservations, per-object write epoch, readiness during async activation, serialized effect cleanup, dependency ordering, last-live state and bounded exclusive-only drain. + +--- + +## 5. Transactional-document Chord v2 / p3 target design: full review + +### 5.1 Blocker: publish-before-migration let new code touch old document shape + +**Status: Later target text changed toward migrate-on-access; verify current implementation plan preserves the fix.** + +The reviewed §15 published new code synchronously, then performed migration commits. If an old transaction held the object's line, new v2 code could synchronously read v1 or draft v2 ops onto a v1 base before migration. + +**Trace.** Old `Draft` v1 transaction awaits storage commit. Publish installs v2. New `snapshot(Draft)` returns v1; new `save` writes v2 paths. Migration then processes a mixed value. Exposed state could send v1 under a v2 contract. + +**Recommendation.** At publication mark bumped documents migrating. New-code reads and transactions wait on the object's line until migration commits, or migrate in memory synchronously and fail/rebase old transactions. Exposed readables keep old snapshot until reset. + +### 5.2 Major: old code could drive a newly replaced native + +**Status: Decided / adopted.** Later design says handles belong to generations and old handles throw stale. + +A shared by-name resource handle returning the latest value lets old job logic silently operate a semantically new sandbox/socket. + +**Recommendation.** Per-generation handles; old handles revoke on replacement. New calls wait during replacement. + +### 5.3 Major: routine serialization is required + +**Status: Decided / adopted in §11.** + +Queue item X can be processed by the old handler and by the new reconciler after reload. Search and subagent reporting can duplicate or reorder actions. + +**Recommendation.** Stable per-routine dispatcher; new reconciliation starts only after old handler/cleanup settles, with a routine-local deadline. + +### 5.4 Major: old code's final transaction can lose already-completed work + +**Status: Decided / addressed by two-phase jobs; see section 8.** + +A shell completes, then its old-code transaction touches a migrated result document and throws. Cleanup could falsely record `interrupted`; external effects already happened. + +**Recommendation.** Old run returns outcome to runtime; current code applies it. Mutating methods need idempotency/request ids or non-retryable stale semantics. + +### 5.5 Major: immediate exclusive replacement under dependent work + +**Status: Resource design evolved to `replace` handoff; no-wait replacement remains an explicit tradeoff.** + +The Durable harness/execution dependency illustrates that work in other objects may close over the native. Closing under it differs from restart, where old work dies. + +**Recommendation.** Propagate replacement through declared deps; old handles stale; bounded abort/wait for truly exclusive natives if native corruption is possible. + +### 5.6 Major: Durable phases pin registry tables + +**Status: No-late-binding immutable tables adopted. Open issue remained for resource-bumped old entries.** + +Pinned old tools either need old declarations kept alive, or stale must terminate and reschedule the Durable invocation without committing a tool error. + +### 5.7 Major: start routines and entries only when contributor is live + +**Status: Decided / adopted as readiness gates and old-entry retention, with later review caveats.** + +Publishing an Extension before resources open can resume a Durable task into `not_ready` and commit the fault. + +**Recommendation.** Contributor entries become current only at readiness. Existing entries remain until success or failure, and are removed on failure. + +### 5.8 Major: jobs must count for keep-alive and relocation + +**Status: Decided / adopted in job section.** + +A tab can close while shell/codemode work remains. If jobs are not activity, idle stop or relocation kills/moves beneath the job. + +**Recommendation.** Host activity includes running/unsettled jobs; relocation refuses or explicitly aborts them and waits through finish. + +### 5.9 Minor: wire atomicity does not compose with adapters + +Raw v2 document ops cannot apply to an adapter-produced v1 view. Atomicity per hold can also tear a commit observed through two holds. + +**Recommendation.** Diff adapted values per binding; define whether atomicity is per hold or batch all holds on a link/object. + +### 5.10 Minor: persisted identity and one-way migration footguns + +**Status: Stable-identity restriction, tags and storeKey were later adopted. Revert behavior remained a design concern.** + +Worker generation and owner access path make principal keys unstable. Session clone changes ids. Removed types leave persisted docs without an owner. Lazy one-way migration means some docs advance and dev revert cannot read them. + +**Recommendation.** Persist only host-lifetime objects or types with explicit stable `storeKey`; retire by tag/type; retain prior base until a subsequent successful reload or explicitly accept forward-only migrations. + +### 5.11 Contradictions and startup cycle + +Earlier plugin-loading text said registries had no versions/adapters while target design defined them. A worker readiness/report cycle was possible if server routing waited for report while `host.ready()` needed a server object. + +**Recommendation.** Report code installation separately from object readiness; route per live object; keep docs aligned. + +### 5.12 Verdict on transactional documents + +Transactional documents, object lines and copied Durable transaction/eviction semantics are a sound and simpler base than drain handover. Before implementation, the design needed to close the publish/migration gap, preserve per-generation native handles, serialize routines, finish jobs through current code, gate entries on contributor readiness, count jobs as activity, and make rollout status/revisions precise. + +--- + +## 6. Jobs design + +### 6.1 Real-case classification + +- Chat `!cmd` shell: detached background work is required. It must outlive the tab, expose progress to multiple viewers, be abortable by id and land exactly once in the transcript. +- Login OAuth: detached job is required. A short link blip must not abort the provider flow; prompts are Questions; notices are progress. +- Env test: a long method call is enough. It is request/response, idempotent and may die with its caller. +- Codemode/tool execution: Durable task, not Chord job; it already needs checkpoints/replay/task ownership. + +**Owner decision: adopted in current §10 / Questions design.** + +### 6.2 Initial concrete job proposal + +The proposal was a runtime-owned job record: + +```ts +type JobEnd<A, R> = { id: JobId; args: A } & + ({ kind: "done"; outcome: R } | + { kind: "failed"; error: string } | + { kind: "lost" }); +``` + +A versioned definition provides typed args, progress and outcome, optional persistence/concurrency/retention, and pure upgrading of old results. Starting belongs in a transaction (`tx.start`) so busy check and request-key dedupe are atomic. `ctx.progress` is throttled/in-memory. Running/unsettled records count as activity. Persisted running records from a dead incarnation become `lost` on next activation. + +The initial proposal separated `settle` (transactional current code) and `deliver` (idempotent external I/O). The owner then preferred the cleaner two-phase form below. + +### 6.3 Owner's two-phase `run` / `finish` shape + +**Status: Decided / adopted.** + +I agreed with the shape: `run` returns the result; `finish(end, tx)` from current code records it; Chord acknowledges the run in the same transaction. This is cleaner than separate settle/deliver for ordinary outcomes. + +Required refinements I recommended: + +1. Typed terminal end including `lost`. +2. Atomic start inside a transaction. +3. Pure `accepts`/upgrade for args/results from an older job version; terminal stored results are not rewritten. +4. Explicit rule for external systems. +5. Key tombstones retained long enough to dedupe client retries. +6. Pending finish must retain outcome and retry after reload/activation. + +### 6.4 Problem 1 trace: result document migrated during run + +1. `tests` migrates v1 -> v2 while v1 env test runs. +2. v1 `run` returns and writes no document. +3. v2 current `finish` drafts the v2 document and records outcome. +4. Chord acknowledgment and app document write commit atomically. + +This solves the original false-`interrupted` bug. + +### 6.5 Problem 2: external systems cannot be written inside finish transaction + +**Decision/recommendation.** The external operation belongs in `run` and carries `ctx.jobId` as its dedupe key. For the shell, Durable submission uses `requestId = jobId`. `finish` only records app state/acknowledges. + +Crash windows: + +- Submit succeeds, process dies before acknowledgment: persisted run appears `lost`; Durable already has exactly one entry because request id dedupes. +- Process dies before submit: outcome is lost. This cannot be solved for non-replayable work. +- External system without a dedupe key: at-most-once attempt in `run`, or an outbox document written by `finish` and drained at-least-once by a serialized routine. + +### 6.6 Abort and the missing closing signal + +**Open/review point later found in current §10.** `ctx.signal` is fired by user abort. A cancelled shell still must submit its cancelled transcript entry, but a Durable submission using that signal rejects immediately. + +**Recommendation.** Add `ctx.closing`, which aborts only on forced object/host closure. The run stops the process with `ctx.signal`, then performs final idempotent delivery under `ctx.closing`. Running jobs keep their object alive. Forced disposal records `aborted` and runs finish before closing the store when possible. + +### 6.7 Busy semantics and finish failure + +**Open finding from current §10 review.** Current text counted runs whose `finish` had not committed against `concurrency`. A migrated document without a valid migration, a broken finish, or missing job adapter could block a concurrency-1 shell forever. + +**Recommendation.** Concurrency counts only live effects. An ended record waiting for finish remains visible as `{phase:"finishing"}` and reports failure, but does not hold the live-run slot. If the owner intentionally wants ordering through finish, make that separate/explicit. + +### 6.8 Progress and unavoidable limits + +- `OutputBuffer` + `Progress` avoids write amplification; progress is in-memory and may lose the tail on crash. +- Old-version live progress should be hidden unless adapted. +- A command that completed just before crash can return as `lost` despite external side effects. No generic runtime can distinguish this without effect-side idempotency/receipt. +- A process cannot be relocated; relocation must refuse or abort. + +### 6.9 What to drop from the old §10 + +- `cleanup(jobId)` that could overwrite a completed result with `interrupted`. +- The old run's final `self.tx`. +- `running.value.length` followed by `start` (non-atomic). +- Treating a request id merely as run id; use it as a dedupe key while Chord mints globally unique run ids. +- “After crash nothing runs” as the only behavior; persisted records allow a `lost` outcome. + +### 6.10 Current §10 review points + +**Status: some open.** + +- A blocked/failing finish must not hold the busy slot forever. +- The abort path needs `ctx.closing` and explicit object-ending semantics. +- Job `accepts` must preserve args/outcome meaning and never silently drop blocked outcomes. +- `finish` retry/backoff must retain the terminal result. +- Running and unsettled runs count as activity. + +--- + +## 7. Questions service + +### 7.1 Overall assessment + +**Status: Decided / adopted as a p2 service, not a Chord/Durable primitive.** + +The design is sound if the in-memory pending store is the truth, calls use stable keys to reattach after link loss, and authorization is per question. + +### 7.2 Open list must derive from resource state, not a document + +A migrated/persisted open-list document can retain ghost questions after resolver loss or old-generation cleanup failure. + +**Recommendation adopted in questions.md.** Store `{question, resolve, reject}` in a resource Map; expose a filtered readable/derive. Answer/abort/deadline atomically `take(id)` before resolving. A late/second answer gets gone. + +### 7.3 Link loss is not denial + +Durable `beforeTool` catches a hook error and blocks the tool. A phone network blip during approval must not become a permanent `blocked: unavailable` transcript result. + +**Recommendation adopted.** `ask(...,{key})` takes stable flow identity (task id/job id). Retry `unavailable` under the caller's signal; same key reattaches. + +### 7.4 Reattachment must include prompt identity + +**Open finding from current Questions review.** Login example used `key: ctx.jobId` for successive prompts. If one OAuth flow asks account selection and then manual code, the second ask can attach to the first question unless key includes prompt id/counter or reattach validates exact question identity. + +**Recommendation.** Stable key is `(flow id, prompt id)`; reattach only if scope, audience, policy and normalized question match. + +### 7.5 Authorization and secrets + +**Adopted.** + +- Worker asker only in own session. +- Secret questions only from server askers. +- Audience controls answer authority; guest send access does not imply tool approval. +- Viewing and answering checked separately. +- Secret answer never enters readable/doc/log; redaction occurs in answer before provider code receives it. + +### 7.6 Tool approval memo must bind approved arguments + +**Open finding.** A bare boolean memo can approve different final arguments if argument-rewriting hooks change after reload/crash before intent commit. + +**Recommendation.** Memo `{allowed, digest(finalArguments)}`. Re-ask if digest differs. Approval hook must run after all argument-rewriting hooks. + +### 7.7 Approval needs a default deadline + +**Open finding.** Pending `ask` is an active call. With owner offline and no deadline, it can hold worker keep-alive and relocation fence indefinitely. + +**Recommendation.** Tool approvals get a bounded default deadline that resolves to deny/no-answer. Long-lived multi-party question tool already specifies one. + +### 7.8 Plain-data resource required for generation GC + +**Open finding.** `replace: old => old` on an instance of `PendingQuestions` retains the old class prototype and bundle generation, contradicting loader D reclamation. + +**Recommendation.** Resource value is a plain Map/listener container; current facet code supplies all logic. If a class is unavoidable, replace constructs a new-generation instance from old data. + +### 7.9 Notices are not questions + +**Adopted.** Login auth URL, device code and provider notices are one-way job progress, not prompts. Remote device-code auto-selection stays in the login flow. + +### 7.10 Durable question-tool context example + +**Open doc issue found in review.** The question-tool scenario used `ctx.get` / `ctx.toolCallId` as if Durable tool execute received a Chord ctx. Durable provides its own API/context. + +**Recommendation.** Wrap the call in `host.root`, use Durable task/call id as the question key, and document that unsafe tools interrupted after execute intent do not re-run hooks. + +--- + +## 8. Resources with `replace` + +### 8.1 Hand-over hook rationale + +**Status: Owner adopted.** A synchronous `replace(old)` may transfer a native/container into the new shape without closing it. Old-generation handles stale. Async behavior/handshakes live in routines. + +### 8.2 `replace(old) => old` generation leak + +See Questions 7.8. More generally, transferring a value with old prototype/method closures keeps old generation code alive. + +**Recommendation.** Resource values should be native/plain data; behavior outside. `replace` constructs a new wrapper if wrapper code changed. + +### 8.3 Dependency semantics were underspecified + +**Open review point.** When a provider is handed over with `replace`, are dependents kept, handed over, or reopened? Reopening a dependent whose close owns the same socket can close the transferred native. + +**Recommendation.** State explicitly: a provider's `replace` preserves its identity for dependency comparison only if declared; each changed dependent either has its own `replace` or reopens without closing transferred provider state. Close/open ordering follows deps. + +### 8.4 Replace failure and old work + +If replace throws, design closes old value while old jobs/calls still use it. This is an accepted restart-like native error, but must be explicit. Old handles stale immediately and object fails; no code may silently follow the replacement. + +--- + +## 9. Routines and naming + +### 9.1 Required semantics + +Standing behavior is separate from jobs, methods, docs and resources. It has `on`, `every`, `loop`, `custom`; errors are caught; cleanup and in-flight handler serialize across reload; timer due date survives reload. + +**Status: Adopted as `self.routines`.** + +### 9.2 Naming recommendations + +Initial ranking: + +1. `self.standing`: directly matches “standing behavior”; no collision. +2. `self.supervised`: Chord starts/stops/restarts/catches errors, analogous to Erlang/OTP and Akka supervision. +3. `self.daemon`: background behavior for object lifetime. + +Related framework terms: RxJS subscriptions; Kubernetes controllers/reconcile loops; Orleans timers/reminders; Durable Object alarms; Temporal durable timers; React setup/cleanup effects. + +### 9.3 `self.run` versus `self.routines` + +I ranked `self.run` below `routines`. + +- Job execution and job callback already use “run”. +- `self.run.custom(... ({run}) => ...)` gives the same word two meanings in one call. +- As a verb, `self.run.loop` sounds imperative rather than declarative. +- `self.routines.every(...)` names the category, has no collision, and reads consistently. + +**Later status: current target used `self.routines`.** + +--- + +## 10. Reload as restart: dropping trials and dry runs + +### 10.1 Position + +I agreed with dropping the old prepare trial machinery and migrate dry run because the old bugs came from trial objects acquiring refs/opening resources/racing activation, and dry run covered only loaded documents. + +I initially recommended retaining a cheap recording-only factory call before publication to catch common factory typos without creating objects or resources. The later owner decision explicitly chose **no factory trial**; reload is explicit, not on save, and factory failure is restart-like. Treat that recommendation as **not adopted**. + +### 10.2 Facet setup failure semantics + +Recommendation at review time: + +- Failed facet object; served objects fail with it; entries leave table as after restart. +- Build registry tables once after setup staging. +- Contributor entries appear only when contributor is live. + +Later owner decision changed this: a throwing setup retains that facet's previous version while other facets publish. This introduced per-plugin mixed-version/revision concerns recorded in section 12. + +### 10.3 Long publish + +Only objects of changed plugins should rerun factories. Measure/report synchronous publish duration. Do not lazily rerun factories, because calls would observe mixed old/new code by object. + +**Status: Adopted: publish is measured/reported and one tick.** + +### 10.4 Start barrier + +Recommendation: first start waits until required host objects settle; optional failed facets remain unavailable; Durable resumes only after required cwd/model/config are ready. Reload has no global barrier. + +**Status: Adopted in broad form.** + +### 10.5 Exposed singleton migration failure + +I objected to merely refusing one access while references supposedly remain hydrated. A singleton exposed under a new contract but still v1 cannot truthfully remain healthy. + +Recommendation then: migrate loaded exposed singleton during after; on throw either fail the member/object with an explicit error state and no v2 ops, while preserving stored data. The later text chose per-member error while methods/other members remain live. Verify wire/hydration semantics make that state explicit rather than serving stale shape. + +### 10.6 Jobs of failed objects + +Their terminal outcome remains recorded; `finish` waits until current code exists after retry/reload. Pending finish should not hold the live concurrency slot forever. + +--- + +## 11. Current jobs / routines / resources / Questions consistency review + +### 11.1 Finish-pending busy slot + +Current §10 said `running` includes finish-pending runs and the busy slot remains held. A broken finish or missing adapter can permanently block concurrency-1 work. + +**Fix recommendation:** separate `running` effect from `finishing` outcome; preserve/report result and retry finish without blocking new effect unless explicitly requested. + +### 11.2 Abort/final delivery missing `ctx.closing` + +Current §10 says abort fires `ctx.signal`, but shell example submits externally after process end. Cancellation aborts the delivery too. + +**Fix recommendation:** job ctx has stopping signal and closing signal; object/job lifecycle explicitly says when finish may run. + +### 11.3 Tool approval memo argument digest + +See 7.6. + +### 11.4 Question store class generation leak and key collision + +See 7.4 and 7.8. + +### 11.5 Resource replace/deps wording mismatch + +See 8.3; update resource, reload and version table consistently. + +### 11.6 Durable context mismatch in question-tool example + +See 7.10. + +--- + +## 12. Focused reload and rollout review of §15 + +**Status: Open unless current target has since changed.** These findings apply to the explicit load -> publish -> after, restart-like rollout reviewed at 6559589bd. + +### 12.1 Major: Durable phase spans a resource bump + +A pinned old registry table invokes old code after the contributing resource is replaced; old handle throws stale and Durable can commit the tool error permanently. + +**Concrete p2 trace.** Codemode extension reads `sandboxResource.value`; long agent phase pins Extensions; reload bumps sandbox; next script in phase uses old entry and gets stale. + +**Recommendation.** p2 Durable bridge treats stale from a pinned entry as invocation loss without commit, then reschedules with a fresh snapshot. Correct “old, consistent code” wording: it is consistent only while declarations it uses stay alive. + +### 12.2 Major: old entries must leave when new contributor fails + +Old entries staying “until new version is live” has no end when factory/resource/migration fails. They remain callable against closed/stale handles. + +**Recommendation.** Retain old entry until new contributor is live **or failed**; remove and report on failure. + +### 12.3 Major: overlapping `after` steps across quick reloads + +R1 is reopening port 9090. R2 publishes before R1 finishes and starts another reopen. Opens race (`EADDRINUSE`), R1 may complete after R2 close and leak, and R1 may open R2's gate incorrectly. + +**Recommendation.** Per-object serialized after queue. Superseded async opens close their eventual value and cannot mutate readiness. + +### 12.4 Major: one host revision cannot represent partial facet retention + +A setup failure keeps plugin A at R-1 while plugin B reaches R. The host is at neither revision. A host-wide mismatch status misclassifies unaffected contracts and reconnect state. + +**Recommendation.** Revision/status per plugin (or complete code-set digest plus per-plugin exceptions). Mismatch carries the serving plugin's desired/running revision and status. + +### 12.5 Major: client wait has no exit when its own install rejected + +Provider is ahead, so browser assumes its own install is coming. If web facet setup rejected or tab never installs, calls park to deadline, reacquire and park again. + +**Recommendation.** Binding checks local install status; rejected fails fast. Browser/TUI unable to install in place falls back to full page/worker swap. + +### 12.6 Major: server incarnation forces unnecessary full reinstall + +Server restart changes incomparable incarnation although code digest is unchanged. Every worker reruns all factories/routines and cannot serve until reporting, stalling all sessions. + +**Recommendation.** Revision also carries complete code-set digest. Same digest means synchronized without install; incarnation/sequence orders rollouts. + +### 12.7 Minor: routine start ordering can cycle with readiness gate + +Old serialized routine handler calls its own object and parks behind readiness. New routine waits for old handler. Gate opens only after new routine starts. + +**Recommendation.** Open readiness after resources/derives are ready, then start routines. Routine can reconcile once calls are admitted. + +### 12.8 Minor: plugin-loading removal check contradicted restart-like bind failure + +Plugin-loading §4.2 still refused removal while holders exist, while new §15 removed consumer checks and says consumers fail at bind. Durable live-task removal also has special behavior in plugin-loading §10. + +**Recommendation.** Pick one rule and align docs. Under restart semantics, delete holder refusal and explicitly point removal to Durable task provenance/stop policy. + +### 12.9 Additional rollout review points considered + +- Worker spawned mid-rollout must install the server's complete desired code set before routing; snapshot desired set atomically with spawn. +- Rapid R1/R2 should reconcile directly to latest complete set; superseded installs cannot publish late. +- Background/old clients need a full-reload fallback after install deadline. +- Server load/build error before publish should stop rollout and preserve previous desired revision everywhere. +- Calls parked on a binding released by consumer code replacement end and are not resent: this avoids duplicate method execution. +- A facet retained at old version may conflict with newly published consumer facets/contracts/registry owners on the same host; per-plugin status must make failures local and explicit. + +### 12.10 Verdict at that review + +Single-host load/publish/after was judged sound in shape, but not ready until Durable stale handling, failed-entry removal, per-object after serialization, per-plugin revision/status, client rejection exit and digest-based reconnect were specified. These fixes do not reintroduce factory trial or drain. + +--- + +## 13. Prototype regression-test audit summary + +These throwaway tests were written under `/tmp/chrev` during the reviews. They are historical and may no longer apply to current code: + +- `reentry.test.ts`: old code after member drain / lost write. +- `registry.test.ts`: registry call ownership. +- `queued.test.ts`: queued run closure ownership. +- `doubleswap.test.ts`, `doubleswap2.test.ts`: effect cleanup overlap/stalled flag. +- `hang.test.ts`: unbounded post-drain step. +- `failedblocks.test.ts`: failed object blocking unrelated reload. +- `revert.test.ts`: failed handover unable to revert. +- `r2facet.test.ts`: facet removal under work; multi-member stall. +- `r2registry.test.ts`: object-valued entries; no-context self-wait. +- `r2activation.test.ts`: prepare/activation deadlock. +- `r2gc.test.ts`: facet generation reclamation (passed). +- `r2seq.test.ts`: sequential members causing needless call timeout. +- `r2revision.test.ts`: rejected delta skipped after later revision. +- `r2entryabort.test.ts`, `r2entrystale.test.ts`: registry context/signal/stamp. +- `r2init.test.ts`: init mutating live previous value. + +--- + +## 14. Cross-cutting recommendations retained across designs + +1. **Generation ownership must be explicit.** Handles, routines, jobs and pinned registry values must either retain the exact declarations they use or fail in a way the caller can safely replay. +2. **Do not call partial external work generically retryable.** Retry requires a request/dedupe key or a durable outbox. +3. **No old-code write through new shape.** Transaction/document version checks are necessary, including copied-value/cross-document decisions where possible. +4. **Serialized standing behavior.** New reconciler starts after old handler/cleanup for the same routine; no object-wide drain. +5. **Contributor readiness gates registry visibility.** No task/tool/route should become discoverable before its resources and exposed state are ready. +6. **Jobs are activity.** They keep objects/workers alive and participate in relocation/stop policy. +7. **Immutable registry tables need lifecycle ownership.** Pinned old entries are safe only while the generation's declarations stay usable. +8. **Revisions describe actual code, not intended code.** Partial installs require per-plugin status; reconnect equality needs code-set digest. +9. **Complete-set reconciliation beats deltas.** Rejected revisions must not be skipped by a later delta. +10. **GC is part of correctness.** Resource values handed over across bundle generations must not carry old prototypes/closures unless intentionally pinned. + +--- + +## 15. Known owner decisions recorded in this conversation + +- **Rejected:** drain-based handover, distributed re-entry/root ids, closure-wide drain machinery. +- **Rejected:** pure version-fence design without transactions. +- **Adopted:** Chord-owned transactional documents modeled after Durable, per-object lines and serialized host storage commits. +- **Adopted:** handles belong to code generations; old code touching migrated documents or replaced handles fails. +- **Adopted:** routines serialize per routine across reload. +- **Adopted:** resources support synchronous `replace(old)` handoff plus dependency order. +- **Adopted:** versioned immutable registry tables; no late binding. +- **Adopted:** typed host values and start barrier. +- **Adopted:** jobs use intent/effect/outcome (`tx.start`, old-generation `run`, current-code `finish`) with atomic acknowledgment. +- **Adopted:** abort is a request; live progress uses `OutputBuffer`/`Progress`; jobs count as activity. +- **Adopted:** Questions is a p2 service backed by an in-memory shared resource and per-principal Chord views. +- **Adopted:** explicit reload, never on save; no factory trial, no migrate dry run, no prepare-time live-consumer compatibility vote. +- **Adopted:** facet setup declarations stage at publish; setup failure keeps that facet's previous version while other facets may advance. +- **Adopted:** explicit load -> synchronous publish -> ordered after activation and readiness. +- **Adopted:** restart-like contract mismatch behavior at bind. +- **Adopted:** rollout revisions identify complete desired code sets and installs reconcile missing code; later review found partial-facet status still needs per-plugin precision. +- **Naming adopted in current text:** `self.routines.on/every/loop/custom`; I preferred it over `self.run` because `run` collides with job runs and custom's helper. + +## 16. Still-open points from my reviews + +Unless superseded after the conversation, the main open points were: + +- Whether finish-pending jobs hold concurrency slots; recommendation: no, expose `finishing` separately. +- A separate `ctx.closing` signal for final delivery after user abort. +- Pure adaptation of old job args/results and behavior when no adapter exists. +- Question key identity for successive prompts and argument-digest binding for tool approval. +- Default approval deadlines and relocation/keep-alive behavior. +- Plain-data Question store to avoid generation leaks. +- Exact resource `replace` semantics for dependents and failure. +- Durable handling of stale pinned registry entries. +- Old-entry removal when contributor activation fails. +- Per-object serialization/cancellation of overlapping after phases. +- Per-plugin rollout revision/status and code-set digest on reconnect. +- Client install rejection/full-reload fallback. +- Alignment of plugin removal semantics between target-design and plugin-loading. +- Explicit wire behavior for adapted document ops and cross-hold atomicity. +- Revert/forward-only policy for lazily migrated persisted documents. diff --git a/apps/p2/docs/p3-reviews/rev-ho-sol.md b/apps/p2/docs/p3-reviews/rev-ho-sol.md new file mode 100644 index 000000000..0896c3f0b --- /dev/null +++ b/apps/p2/docs/p3-reviews/rev-ho-sol.md @@ -0,0 +1,811 @@ +# rev-ho-sol — Adversarial target-design, reload and Chord v2 reviewer + +## Scope, provenance and status + +This is the recoverable record of my findings, review points, open questions and recommendations in this conversation about the target design, reload design, Chord v2 and its p3 successor. It is not the functionality map. It records historical proposals as historical proposals, not as recommendations to restore an obsolete design. + +The conversation was compacted. Exact earlier wording survives for the two-phase-jobs response and Questions review in scratch files; those are included verbatim below. Earlier points known only from the retained conversation summary are reconstructed and explicitly labelled. The adversarial plugin report and its essential migration sketches also survive and are included verbatim. I cannot recover missing pre-compaction wording or infer owner acceptance from my own recommendation. + +Statuses: + +- **Owner decided:** explicitly identified as an owner decision in the retained conversation, or an established rule of the reviewed target. +- **Open:** recommendation or question with no subsequent owner decision known to me. +- **Partly decided:** the architectural rule is decided, but the concrete migration or correctness issue remains open. +- **Superseded:** an earlier recommendation or expectation was replaced by a later owner decision. It must not be treated as current advice. +- **Historical/unverified:** a candidate or old reproduction whose outcome/current applicability is not established by the surviving record. + +The last full adversarial review used `pi-private origin/private` at `6559589bd66bc909cfc1d694426799a3c4cd95f6`. Earlier referenced review points used `320ee90511eaf74624092c164ee13b5650868f9f` and older snapshots, including `f04acc2d`. File/line citations in copied reports refer to those snapshots, not necessarily the present worktree. The target design was not implemented/executed by the plugin review: this was source inspection and migration sketching. + +Only this review file is being written for the collection request. No commit is requested or made. + +## Owner decisions known from this conversation + +1. **Owner decided — transactional documents.** Objects are behavior; documents own JSON data. Within a transaction the only permitted await is `tx.draft`. Do not call other objects, models, processes, network operations or a nested transaction there. Fetch external input beforehand and decide writes inside the transaction. +2. **Owner decided — generation-bound document handles.** Old code accessing migrated data throws rather than receiving a new shape through an old type. An old subscription ends with `stale`; partial transactions do not land. Do not weaken this rule to rescue an inadequately checkpointed application phase. +3. **Owner decided — non-draining ordinary reload.** Ordinary running calls/jobs keep their old code. Reload neither waits for nor signals them. They may fail on migrated documents or replaced natives. Cleanup of deliberately removed/replaced native facilities is domain-specific, not a return to generic distributed draining. +4. **Owner decided — declaration staging, no speculative execution.** Load/publish/after replaces speculative factory trials and migration dry runs. Facet setup runs once with declarations staged; native activation and lazy migrations can fail after publish. Check dependency cycles. Contract incompatibility is a bind-time failure with a useful error, not speculative invocation of live consumers. +5. **Owner decided — failed facet retains its previous version.** If setup throws, discard that attempt's staging and retain that facet's previous services, entries and facet object. Other successful facets can move forward. My earlier recommendation to withdraw all old declarations on such a failure is **superseded and withdrawn**. +6. **Owner decided — immutable registries and phase snapshots.** Registration tables are swapped atomically. Already-pinned Durable phases retain consistent old code; do not introduce late-bound registry lookup to silently switch implementations after an await. +7. **Owner decided — start barrier.** Open Durable without scheduling, supply typed host values, install required contributors/configuration, check cwd/model, then resume. A model or cwd check must not race configuration initialization. The remaining open issue is identifying required persisted policy contributors, not whether a barrier exists. +8. **Owner decided — jobs use intent/run/current-code finish.** The author writes `run` and `finish`; the runtime captures a result receipt independently of application document generations. Application finish writes and acknowledgement are atomic. Pure version adapters are required for older inputs/results/progress; unsupported receipts remain blocked and observable. +9. **Owner decided — abort is a request.** Actual native settlement determines the result. A success remains success even if the signal is aborted; partial output must be retained. An uncooperative run stays active until it settles. Finish is not run in an already-aborted caller/run context. +10. **Owner decided — external execution has an uncertainty boundary.** Arbitrary commands cannot be promised exactly-once execution. A persisted execution intent with no recorded outcome after a crash is lost/unknown; do not automatically rerun it. A keyed Durable submission/outbox can guarantee submission admission deduplication, not exactly-once network calls or arbitrary shell effects. +11. **Owner decided — no generic call/root IDs, no stringly typed replacement API.** Keep typed job definitions/contracts and existing domain request IDs. Do not grow Chord into another general phase-based workflow runtime; resumable/checkpointed work belongs in Durable. +12. **Owner decided — Questions is a p2 service.** Use a shared authoritative broker resource with authorized per-principal facades, not a new Chord or Durable primitive. Session send permission is not an approval grant. Scope strings and display names are not authority. +13. **Owner decided — native values are not revoked by a proxy.** Stale handles do not revoke a value already borrowed from a resource. Read `.value` at use; do not keep extracted natives/listeners across an await and then expect a handle fence to protect them. Handover and shutdown still need domain-specific handling. Listener behavior belongs in routines. +14. **Owner decided — dependent resources and derived sources need explicit rebinding.** Declare native dependencies. Providers/dependents are replaced in the specified order; a dependent closes only what it owns. A derive must name its resource handle rather than capture the old native source in its factory. +15. **Owner decided — standing work is grouped and serialized across replacement.** The retained conversation records the preferred name `self.run.on / .every / .loop / .custom`; the cited `6559589bd` files and my verbatim sketches still spell it `self.routines`. Naming drift is historical, not an additional semantic recommendation. New routine instances wait for the previous handler/cleanup, subject to the documented stuck-handler deadline. +16. **Owner decided — presence uses origin identity and linger.** Two tabs are two origin clients even when they share a principal. Linger keeps a worker alive but excludes a client from a live audience/viewer count. Auxiliary subscriptions do not count as extra people. +17. **Owner decided — explicit retirement and stable persistence identities.** Archive is not deletion. Session deletion retires tagged server documents and session storage/catalog data. Persisted drafts/jobs/metadata must use stable logical ownership, not an owner's access path or a worker generation as the persistent identity. +18. **Owner decided/as-reviewed — loader D and SDK direction.** Plugin reloads are per-facet in place; base/runtime bundle changes restart affected hosts. Host values and registries are versioned. UI components/APIs are not all independently versioned yet. Do not invent unsupported compatibility guarantees. + +## Earlier review points reconstructed from compacted history + +These are reconstructed, not represented as verbatim transcripts. + +### Two-phase jobs and detached work + +**Partly decided.** I agreed with `run` followed by current-code `finish(tx)` as the right default, replacing my broader delivery-callback proposal. The required intermediate boundary is a runtime-owned receipt: atomic admission → effect → outcome capture → finish+acknowledgement. Reload does not rerun effects or preempt an admitted finish transaction. Persistent jobs require stable owners and persistent receipts. The exact surviving response is reproduced below. + +**Partly decided/open integration.** A Chord finish cannot call Durable `conversation.submit`, even if both use SQLite. A cross-store handoff needs a persistent outbox written in finish, drained outside transactions with a stable submission request ID, or authoritative execution/delivery entirely in Durable. Count pending deliveries as work during routine restart/backoff. Submission admission may queue a write rather than place it; observe `submission.wait()` and surface `unanswered` where placement matters. Do not label a completed command as executing solely because delivery is pending. + +**Owner decided principle; example correction/open verification.** The earlier shell example submitted externally before returning its native outcome and inferred cancellation from `signal.aborted`. A failed submission could erase a successful command result. Return and capture actual settlement first, then use current finish/outbox delivery. The old shape was still visible in the cited target snippet; I did not recount this already-discussed problem as a new adversarial finding. + +**Owner decided separation.** Questions.ask and cancellable environment tests can be ordinary long calls when caller-scoped lifetime is appropriate. Shell execution is not browser-owned: it needs detached counted ownership, progress, abort and outcome recording. Login ownership is not identical to one tab's RPC. Do not make every pending interaction a job. + +### Questions broker and authorization + +**Owner decided architecture; open implementation checks.** Principal-keyed Chord objects cannot each hold a separate broker Map. Worker askers and owner viewers must meet at shared authoritative backing. Keep descriptors/resolvers there; expose a readable or update a document through current code. An old ask continuation must not remove a row through a handle whose document migrated. + +**Open correctness checks.** Install a waiter before publishing. Serialize create/answer/cancel so only one settlement wins. Cancellation during publication must not orphan a row. Validate against the stored question; reject duplicate/late answers; use fresh IDs after restart. Never await a human while holding a transaction/broker lock. Resource replacement/close must reject all waiters. An ephemeral list disappears with waiters; persisted open lists must be reconciled before exposure. Accepted input is not a durable-continuation guarantee. + +**Owner decided authorization principle; concrete checks remain required.** Scope addresses cannot authorize actions. Restrict askers to permitted scopes and independently authorize viewing and answering. Use the authenticated initiating owner identity for login, not the display/author name. Never persist, echo or log secret answers. A guest's send grant permits tool actions but does not imply filesystem isolation or authority to approve other work. + +**Open/partly decided — approval identity.** My earlier six-finding review required approval memos to bind the final tool arguments, execution context and policy, rather than store a bare boolean by task. The later design addresses ordering approval after argument-rewriting hooks. I do not know that policy/context invalidation semantics were decided. Define what exactly an approval authorizes, including safe execution retries. + +**Established implementation trace.** Durable runs `beforeTool` in the call phase, then checkpoints execute. A crash while waiting for approval asks again; a crash after that checkpoint retries safe execution without rerunning the hook, while unsafe execution reports interruption. Therefore “it asks again after a crash” is not universally true. + +**Open/unknown later disposition — quorum, timeout and result types.** I previously reported inconsistent timeout/result typing and contradictory quorum settlement, including vacuous `every([])` authorization. Specify empty-audience, membership/departure, timeout and settlement behavior explicitly and make the return type express every possible result. Do not let an empty audience accidentally authorize work. The compacted record does not retain the original exact wording or a subsequent owner decision on these details. + +**Owner decided mechanism; migration check open — Questions source replacement.** A view attached to a replaced broker readable must preserve emitter identity or rebind to the new source. Naming the resource handle in the derive is the decided mechanism. Capturing a native readable in the factory leaves the view on the old emitter. + +### Login migration + +**Open preservation requirements, not requests to add new functionality.** Preserve text/secret/select/manual-code prompts and help text, per-prompt cancellation, distinct prompt keys and accumulated/redacted progress notices. A callback-server success must withdraw the competing manual-code prompt without cancelling the whole login. Auth URLs, device codes and status notices are not all question forms. + +**Established existing lifetime; recommendation remains preservation.** A login belongs to the authenticated initiating owner, survives one tab leaving when another relevant hold of that owner remains, and ends after that owner's last Providers hold disappears. Questions replaces prompt plumbing, not ownership of the full flow. Use a flow-owned controller/resource/job or explicitly choose a different caller-scoped product behavior; do not change lifetime accidentally. + +### Restart-like reload review + +**Owner decided/superseded older machinery.** Avoid speculative factory trials and migration dry runs: callbacks can open natives, mutate external systems or repeat initialization, and a dry run cannot establish rollback safety. Stage declarations; publish once; perform required resource activation afterward with explicit failed-object reporting. Check cycles and publish coherent registry tables. + +**Owner decided.** Provide the Durable harness/native host values before plugins need them but do not schedule work until required configuration and cwd contributors are ready. This breaks the previous startup cycle without lazy scheduling racing ahead. + +**Owner decided.** Exposed migrated-document failures must propagate with a cause, rather than silently returning an old shape under a new interface or converting a substantive failure into harmless-looking undefined state. + +**Superseded.** An earlier recommendation was to withdraw all old entries after failed facet setup. The owner subsequently chose to retain that facet's previous version. Only its staging is discarded; unrelated successful facets can move forward. The adversarial review used the owner's later policy. + +### Native handover and Durable consistency + +**Owner decided rule; domain work remains open.** A `replace`/stale-handle fence cannot revoke already-borrowed values, callbacks or native work. Handover is safe only when the old behavior has been detached or is explicitly settled. Do not re-report that limitation as though the decided no-proxy policy promised revocation. + +**Open migration checks.** Keep schema and implementation snapshots consistent through Durable phases; pin native request/provider behavior where necessary rather than changing it halfway through an auth wait. Distinguish intentionally live credentials/configuration from incompatible implementation/shape replacement. Codemode specifically needs abort-and-settle before a sandbox is closed while the harness survives; a worker restart instead preserves replay behavior. + +## Historical drain-prototype reproduction points + +Status for this whole section: **historical/unverified**, with explicit supersessions below. The surviving role-named scratch file `/tmp/rev-ho-sol/review.test.ts` contains seven reproduction tests. Its old API names (`effect`, drain deadlines, `installed_pending`, exclusive handover) do not define the current target. No retained output establishes each test's execution result, so these are not claimed as current confirmed defects. + +1. A second reload must not start replacement standing behavior before the first generation's asynchronous cleanup settles. **Still relevant principle, now covered by decided per-routine serialization/deadline behavior.** +2. Raw queued run closures admitted before publication were expected to retain resource ownership. **Superseded ownership model:** current ordinary work may continue under old code, but replaced natives can fail and stale document handles throw. Do not restore generic old-resource pinning from this test. +3. An activation blocked in old native open must not indefinitely prevent publishing newer code. **Relevant non-draining goal; open implementation verification.** +4. Retrying a partially failed activation must not rerun an added cell initializer that already succeeded. **Historical cell-API question; current document create/migrate-on-access and storage invariants supersede that exact initializer test.** +5. A surviving consumer reloaded in the same batch must not escape provider-removal safety if its new code still holds the provider. **Relevant removal check:** changed consumers are not automatically exempt, and new holds during prepare must be checked. +6. Registry handlers were expected to delay an exclusive handover until they completed. **Superseded:** the current target does not drain ordinary handlers; they may observe native errors after replacement. Preserve the domain-specific cleanup distinction rather than this generic wait expectation. +7. A newly added host-lifetime object with a failing native open was expected to be prepared before publication. **Superseded by owner decision:** no speculative trials; after-publish activation/resource failure is allowed and reported. Required initial readiness is handled by the start barrier. + +The original test source is preserved at the end of this file for its concrete traces, not as a proposed current conformance suite. + +## Status of the final adversarial review + +The verbatim report below has six main findings. **All six concrete fixes remain open in the surviving conversation; none is known to have been accepted or implemented by the owner.** Their underlying context includes decided mechanisms: + +| Point | Status known here | +| --- | --- | +| Persisted subagent policy/cwd capability cannot disappear on first-load failure | **Open design/integration fix.** Initial barrier and retained old facets are already decided, but do not solve fresh-start optional-contributor failure. | +| Codemode call tracker must depend on sandbox | **Open concrete migration fix.** Dependency ordering and domain abort/settle policy are decided. | +| Scope-exit preflight must include Chord jobs and pending deliveries | **Open integration fix.** Durable exit checks and job activity accounting are decided; the provenance/admission bridge is not known implemented. | +| Durable watch must be reopened per restarted routine or genuinely adapted | **Open concrete migration fix.** Restarting/serializing routines is decided. | +| Models provider callbacks must be recomposed on code-only reload | **Open concrete migration fix.** Keeping native data while moving behavior into routines is decided; installed SDK closures require extra handling. | +| Web send completion must not clear newer shared composer state | **Open concrete migration fix.** Continuing old admitted calls and retaining host UI state are decided. | + +Additional review/migration points in the report and sketches: + +- **Open application migration:** checkpoint Reporter settlement before touching the plugin's migratable registry. A current registry migration may deliberately fault an old deliver phase before handover, leaving the parent unnotified even though the answer survives. Install the new phase layout without simultaneously breaking the registry schema for already-running v1 reporters. This does not justify changing stale protection. +- **Owner decided mixed-facet behavior; implementation check:** retaining a failed models facet while codemode moves on is valid. Compatible contracts continue; incompatible dependents fail clearly. A rejected install report is still a report for reconnect routing; do not require impossible whole-set success before routing compatible services. My investigated mixed-revision “whole host gate” concern was not promoted as a new confirmed contradiction. +- **Open adapter detail:** the server's worker-generation check cannot be weakened from synchronous validation+write into “sample generation, await, write.” The sketch's proposed `withCurrentWorker` holds the per-session lifecycle gate through the short metadata transaction, or an equivalent transactional authority must be used. Same-process plugin generations are not the worker principal generation. +- **Open data migration detail:** persist per-session child rows with SessionTag; retain only a volatile aggregate for live UI. Bootstrap must not overwrite a concurrent fresh report, create rows merely by reading deleted/missing sessions, or make the UI cache authoritative for relocation. Await authoritative ChildStore.get(session) for that decision. Prune the aggregate when catalog session IDs change. +- **Open consumer migration:** removing ModelConfig.resources requires replacing login's direct native access with typed owner methods and updating sessions/login consumers. Do not remove the member without preserving their operations. No native values should cross plugin boundaries. +- **Owner decided boundary; open helper implementation:** a facet factory cannot self.get its own served types or a remote type. Obtain the subagents reporting reference through the routine's ctx.get, rather than recreating the removed carrier's remote factory hold on the facet. +- **Open lifetime integration:** pending outbox delivery must remain activity while its routine is stopped, restarting or backing off. The illustrative liveness.call around a drain invocation alone is insufficient. +- **Open configuration migration:** initial load must complete before first resume, not merely start in an asynchronous routine. Later invalid files retain last-good validated data and expose bounded errors. Code-only callback recomposition must still use last-valid parsed config if the current file is invalid. +- **Owner decided native-error boundary:** do not wait for every old stream on ordinary reload. Compatible recomposition replaces the pointer for new invocations and lets in-flight operations retain their own old SDK generation. Resource close preserves native write/lock cleanup, not a generic drain of all running calls. +- **Open source error handling:** a fresh per-routine watch/source adapter must surface unexpected termination rather than freeze a last snapshot invisibly. Keep snapshot/ops coherence and explicit source rebinding. +- **Open verification, not another finding:** active presence must count distinct client origins, not object count or auxiliary subscriptions; per-hold linger is excluded from Questions audiences but included in keep-alive. +- **Open performance verification:** preserve structural sharing, bounded windowing, paged history, bounded output and throttled deltas. Do not clone or transactionally rewrite the full transcript on each token. + +Correction to the verbatim sketches: their prose says “64-entry remote window.” That numeric statement was my mistake. In the reviewed `plugins/chat/window.ts`, WINDOW_ENTRIES is **100**, WINDOW_BYTES is **512 KiB**, WINDOW_MIN is **20**, and WINDOW_SLACK is **1.5**, with a tool-result boundary adjustment. The recommendation was to preserve the existing bounded, structurally shared behavior, not replace those limits. The original wording is retained below because this file is an archival review record. + +## Unpromoted investigation questions + +Status: **historical/unverified candidates**, not additional confirmed findings or owner decisions. + +- Can delayed subagent reports race within one worker generation across a facet reload? Check cancellation of outgoing reports, server-side serialization, authoritative generation fencing and aggregate bootstrap ordering. The migration sketch gives a required gate, not proof that today's synchronous implementation already has that race. +- Can read-only/relocation checks disappear in a mixed retained-facet installation? Normal failed setup retains the old entries, so that specific disappearance is not a generic reload bug. Fresh-start failure has no previous facet and is the concrete finding. +- Do pinned extension closures reach mutable Models/configuration in a way that switches implementations after an await? The concrete findings concern callbacks left installed and invocation-time binding; intentionally live configuration is not automatically a snapshot violation. +- Does a kept one-shot Durable watch tolerate stop/start? Source inspection confirmed it does not; this candidate became the watch finding. +- Do SDK/native callbacks retain plugin generations after resource behavior is moved to routines? The provider callback/fingerprint trace became the Models finding; native close/borrowed values are separately governed by the already-decided rule. +- Are in-place UI construction/render exceptions isolated from the host and other slots? Existing slot/TUI factory consumers lack obvious per-entry boundaries. This was considered but not promoted into the final findings; no new owner decision was established, and deferred UI API versioning is not itself proof of an exception-isolation policy. +- Can child metadata or root session configuration be left stale if native configure succeeds and an old continuation then fails? Recommend reconciling authoritative Durable configuration to the registry/sidebar/catalog from current code rather than relying only on that continuation. This is a migration/reconciliation point, not evidence that stale protection should be removed. + +## Verbatim earlier response: two-phase jobs + +Status annotation: the owner-decided two-phase/receipt/abort boundaries above supersede any stale API spelling in this historical response. Outbox integration, placement reporting and migration details remain recommendations unless separately marked decided. + +I agree. **`run` → current-code `finish(tx)` is the right default.** It replaces my broader delivery callback. The necessary refinement is a runtime-owned result receipt between those phases. + +## Contract + +```ts +interface JobHandlers<I, R, P> { + run(input: I, ctx: JobContext<P>): Promise<R>; + finish(done: JobReceipt<I, R>, tx: Tx): void | Promise<void>; +} +``` + +A typed `defineJob` token supplies versioned input/result/progress schemas; `self.job(token, handlers)` binds the current implementation. A receipt contains ID, input, and either returned value, serialized error, cancellation, or unknown outcome. + +Internally: + +```text +atomic admission → run → persist outcome → finish transaction + acknowledgement +``` + +Two author-written phases, but three durable boundaries: admission, outcome capture, finish acknowledgement. + +- `run` returns data; it does not record completion through application handles. +- Chord captures its outcome independently of caller cancellation and application document generations. +- `finish` is selected from current code when its transaction starts. It may only await `tx.draft`; no network, processes, other objects or nested transactions. +- Application writes and the job acknowledgement commit in the **same storage batch**. A failed finish leaves both unchanged; retry finish, never run. +- Reload does not signal or wait for run. An already-admitted finish follows normal transaction semantics; reload does not preempt it. + +Persistent jobs require stable owners and persistent receipts. Keeping jobs only in memory cannot provide restart guarantees. + +## Shell: where local atomicity stops + +```ts +const shell = self.job(ShellJob, { + persist: true, + concurrency: 1, + run: (input, ctx) => collectShell(input, ctx), + finish: async (done, tx) => { + const q = await tx.draft(transcriptOutbox); + q.items.push({ + id: done.id, + conversation: done.input.conversation, + write: shellWrite(done), + }); + }, +}); +``` + +`ShellJob` is a typed definition. Input freezes conversation, command, execution configuration and server-stamped author. `collectShell` uses `OutputBuffer` and publishes through `ctx.progress`, backed by `Progress`: bounded output, throttled/coalesced deltas, no progress document writes. Its return includes the final retained output and execution status, including partial output on cancellation. + +**Calling `conversation.submit` inside finish is invalid.** Even when both stores use SQLite, a Chord transaction is not a Harness transaction. + +Instead, finish atomically records a **persistent outbox item and acknowledges the job**. A p2-owned drainer, outside transactions: + +1. Calls Durable `submit` with a stable request ID derived from the job ID and the frozen write payload. +2. After durable admission, removes the outbox item in a Chord transaction. + +Crash after admission but before removal → retry the same request ID → Durable returns the existing submission. This provides exactly-once submission admission, not exactly-once network calls. For visible transcript placement, track `submission.wait()` and surface `unanswered`; admission may only have queued the write. Queued writes survive conversation abort. + +The drainer must count as pending work for keep-alive and relocation. After handoff, Durable owns the work. UI can retain the outbox preview until placement; it must not call an executing command “running” merely because delivery is pending. A reusable p2 drainer keeps plugin authors at run plus finish. + +## Env test: the shape works directly + +```ts +const tests = self.job(EnvTest, { + persist: true, + concurrency: 1, + run: (input, ctx) => testSsh(input.config, ctx.signal), + finish: async (done, tx) => { + const d = await tx.draft(results, done.id); + d.revision = done.input.revision; + d.outcome = done.outcome; + }, +}); +``` + +Capture configuration once; display results only for the matching revision. Resource cleanup belongs inside run and must not overwrite its actual outcome. + +## Failure traces and required rules + +**Migration:** V1 run starts → result document migrates → V1 returns success → Chord captures the receipt → current finish uses current handles → result and acknowledgement commit together. No false interruption. + +**Schema changes:** current finish cannot safely receive old input/result types by assertion. Require pure version adapters; keep original receipts unchanged. Progress views need the same version fence. Missing adapters or finish bugs leave an observable blocked finish, not a replacement outcome. + +**Abort:** request cancellation and signal run; wait for its actual settlement. A returned success remains success even if cancellation was requested. Do not infer cancellation merely from `signal.aborted`. Finish uses a runtime-owned transaction context, not the cancelled run/caller context. An uncooperative run remains live. + +**Object end:** request cancellation, but runtime outcome capture survives object shutdown. If current code is unavailable, retain the pending finish until reactivation/reinstallation. Explicit session deletion is a separate disposal policy. + +**Crash:** persisted outcome → replay finish safely. Started execution without an outcome receipt → record **outcome unknown**, never automatically rerun an arbitrary command. Recovering its actual output/exit requires a durable execution supervisor. Two phases cannot remove that uncertainty. + +Admission must atomically deduplicate request IDs and enforce concurrency; no separate running check. Executing and unacknowledged finishing jobs count toward liveness/relocation; expose their distinct states. + +## Boundary + +Drop §10’s `cleanup` synthesizing interruption and run-written completion documents. Keep two-phase jobs for one-shot work. Use Durable tasks for checkpointed/resumable work, codemode ownership/cancellation, and preferably shell execution when its authoritative result is the Durable transcript—avoiding the cross-store outbox. Do not add general task phases to Chord. + +## Verbatim earlier response: Questions and ownership + +Status annotation: Questions as a shared p2 service and transaction/stale rules are owner-decided. Concrete broker races, approval binding and login-preservation checks are recommendations; later resolution is unknown unless stated above. + +**I agree with Questions as a p2 service, not a new Chord/Durable primitive.** I also agree with the caller-lifetime distinction, but login’s current lifetime needs qualification. + +## 1. Questions + +Sound with these rules: + +- **Shared backing:** Chord objects are keyed by principal as well as input. Worker-asker and owner-viewer instances otherwise get different Maps. Put the broker in a shared facet resource; expose authorized facades over it. +- **Migration:** keeping resolvers is insufficient. Trace: ask captures `openV1` → document migrates → answer resolves → old ask’s `finally` removes its row through `openV1` → stale error/ghost question. Keep descriptors and resolvers authoritative in the resource; expose a readable, or project into a document through a current-code effect. No retained callbacks that write through old handles. Resource replacement/close must reject every waiter. +- **Races:** install the waiter before publishing; serialize create/answer/cancel with one winning settlement. Cancel during publication must not leave an orphan row. Validate against the stored question, reject late/duplicate answers, and use fresh IDs after restart. Never await the answer while holding a transaction or broker lock. With a resource-authoritative projection, a briefly stale visible row is harmless: answering it returns “no longer active.” +- **Authorization:** scope strings are addresses, not authority. Trusted askers may publish only into permitted scopes; a worker cannot choose another session. Check both viewing and answering separately. Session send access does not automatically authorize approvals. Login scope must use the initiating authenticated owner identity, not the shared display/author name. Never persist, echo or log secret answers. +- **Crash:** an ephemeral open list disappears with its waiters. A persisted list must be cleared/reconciled before exposure; otherwise it advertises unanswerable questions. An accepted answer is not a durable continuation guarantee. + +**Important correction to tool recovery:** current `ToolTask` invokes `beforeTool` in `call`, then checkpoints `execute`. Crash while awaiting approval → hook asks again. Crash after that checkpoint → safe execution retries **without hooks**; unsafe execution reports interruption. “It simply asks again after a crash” is not universally true. Also, later `beforeTool` hooks can rewrite arguments: approval must cover the actual final arguments, not an earlier version. Define whether approval authorizes execution retries. + +Real login prompts fit with an adapter: text, secret, select, and `manual_code` with its help text. Preserve per-prompt cancellation: callback-server success must withdraw a competing pasted-code prompt without cancelling the login. Keep auth URLs, device codes, progress notices, remote device-code preference and redaction; these are not all questions. + +## 2. Do we need jobs? + +Long calls require cancellation propagation and cooperative native operations; cancellation does not undo completed external writes. + +- **Questions.ask:** yes, a long call is sufficient. Its lifetime belongs to the asking flow/task, not whichever viewer happens to be connected. Closing a viewer alone must not cancel a tool’s question. +- **Shell:** not a browser-owned long call. Tab disconnect would cancel it or lose its reply. It needs detached, counted execution, progress/abort, and current-code outcome recording plus reliable transcript handoff. A resource can implement that, but it recreates the useful parts of jobs. +- **Env test:** a long call is enough if caller cancellation and explicit retry are acceptable, returning the result directly. Shared retained results guaranteed across reload need a job’s finish or an equivalent resource/current-code projection. +- **Login flow:** a long call can suffice for deliberately caller-scoped login. Today it survives one tab leaving while another hold of the initiating owner remains; it ends when that owner’s last Providers hold disappears. Preserve that with a flow-owned resource/controller or job, not one tab’s RPC lifetime. Questions replaces prompt plumbing, not flow ownership. + +**Keep jobs for detached run/finish work; do not turn every pending interaction into a job.** + +## Verbatim final adversarial report + +Status annotation: the six concrete findings are open as of this conversation. The report distinguishes them from decided architectural rules and other exercised traces. Scratch paths are provenance, not dependencies required to understand the copied text. + +# Adversarial review findings + +Baseline: pi-private origin/private 6559589bd66bc909cfc1d694426799a3c4cd95f6. Repository untouched. Source inspection, not execution of the unimplemented target. Four plugins: chat (6 files/3300 lines), codemode (3/155), subagents (6/1012), models (14/2381): 29 files, 6848 lines. Full essential facet sketches: /tmp/p2-adversarial-plugin-sketches.md (337 lines). + +## 1. P1 — Persisted subagent policies cannot be optional merely because their facet failed + +Target-design.md:446-447 allows failed optional facets past the initial barrier. Actual writes enumerate whatever ReadOnlyConversations entries exist (plugins/chat/worker.ts:529-534); the one-shot guard exists only in plugins/subagents/worker.ts:55-62. On a fresh worker, subagents setup fails, there is no previous facet to retain, but the session DB still contains one-shot conversations. Chat.send/edit/configure now have no denying entry. This is a write-policy bypass, not a claim that plugins provide filesystem isolation. + +Same-environment cwd relocation is also affected: runtime/worker.ts:488-498 treats missing SessionCwdFollowers as no followers, and :518-525 moves only the root plus returned followers. Persisted ordinary children remain at their old cwd. Server sidebar metadata does not repair that operation. + +Fix: once corresponding persisted state exists, make its write/cwd policy an authoritative required capability before resume and relocation, or move the durable policy invariant into the host. Missing policy must fail closed. A UI notice about an absent extension is insufficient. Keeping old facets solves normal reload failure, not fresh-start failure. + +## 2. P1 — Codemode's tracker must depend on its sandbox + +plugins/codemode/worker.ts:22-33 declares sandbox and calls resources, but calls depends only on harness/worker, not sandbox. Under target-design.md:354-367, bumping only sandbox's native version closes/reopens it without closing calls. Durable explicitly treats that external sandbox abort as undecided and waits for its own call cancellation (packages/durable/src/codemode/codemode.ts:476-485). The tracker never runs abortTask; the script stays running indefinitely. + +Fix: calls.deps must include sandbox, or use a composite native whose close aborts/settles calls before closing the VM. Retain worker-stop versus plugin-removal semantics. This is a concrete missing dependency in the migration, not a request to revoke borrowed natives or introduce generic draining. + +## 3. P1 — Scope-exit protection omits the new Chord jobs and their deliveries + +plugin-loading.md:420-424 records Durable extensions/tasks/tools/entries and refuses or aborts live Durable tasks before removing their definitions. Target-design.md:288 makes Chord jobs activity, but that does not give them exit provenance. Chat's shell is detached work today (plugins/chat/worker.ts:343-350), and becomes a Chord job. + +Trace: remove a batch containing chat and its factory dependents while a user shell is active. There need be zero Durable tasks and no outside factory holder. The documented preflight passes, the conversation type ends (target-design.md:451-452), and the current finish/delivery code disappears. The native command is cancelled or finishes against an ended owner; its receipt/outbox cannot be delivered by the absent plugin. This violates the scope-exit's explicit 'Stop first or abort while definitions remain' policy. + +Fix: include Chord running/unsettled jobs and registered pending delivery work in the existing exit check, with admissions fenced through publish. Refuse or explicitly abort-and-settle before withdrawal. Normal compatible reload still neither drains nor cancels running work. + +## 4. P2 — Chat's kept Durable watch cannot be restarted with its routine + +plugins/chat/worker.ts:158-168 keeps a WatchHandle resource. runtime/durable.ts:107-120 explicitly relies on keeping its subscription across reload; unsubscribe permanently stops it. Target-design.md:315-320 now stops/restarts routines even when a native's version is unchanged. + +Trace: routine cleanup calls watch.stop(); resource is retained; the replacement routine calls start() on that same watch. packages/durable/src/session/observation.ts:193-203 throws 'already started'/'stopped'. Transcript/model updates stop after the first compatible reload. + +Fix: open a fresh Durable watch per routine instance, maintaining a stable host/base source bridge and last snapshot; alternatively add a genuine reusable watch adapter. Do not put the plugin's callback permanently inside a kept resource. Preserve exact ops/window structural sharing; do not turn the full transcript into a rewritten document. + +## 5. P2 — Moving only Models' watcher leaves old provider code installed + +plugins/models/resources.ts:80-85 keeps a plugin-defined proxy; :142-148 skips unchanged fingerprints; :189-208 builds and saves an SDK model generation only when files change. provider-config.ts:263-299,330-362 installs plugin closures in provider auth/stream methods and models.getAuth. + +Trace: move watching to a restarted routine but retain the native SDK Models value and fingerprints. Reload an auth/header/stream composition fix without changing models.json. The new routine skips composition, so subsequent requests still execute G1 provider callbacks and keep G1 reachable. Registry replacement and restarting subscriptions do not replace functions stored inside the SDK object. This differs from the already-decided borrowed-native issue: it occurs on a compatible native reload with no borrowed old handle. + +Fix: retain validated config as data, construct a fresh facade in each setup and force SDK callback composition once per code generation even when fingerprints match. Atomically replace the current SDK generation for new invocations; already-running requests may retain their own old generation. Keep the existing invocation-time binding that prevents auth waits from switching provider implementations. + +## 6. P2 — Old web send continuations can erase a new facet's draft + +plugins/web-shell/web.tsx:34-38 keeps ComposerState above the replaced main slot. src/web/composer-state.tsx:39-49 owns app-scoped setters. plugins/chat/web.tsx:591-606 unconditionally clears draft/files/reply after awaiting send. + +Trace: send A is already admitted; replace chat.web's main component in place; the unchanged web-shell keeps ComposerState; type B in the replacement component; the allowed old A continuation completes and clears B through the shared provider. Unmounting the old component does not invalidate provider-owned setters. Even a cancelled old continuation can overwrite the new component's error/pending UI. + +Fix: acknowledgements compare the captured conversation/request/draft revision, remove only that attempt's attachments, and release only its pending operation. Suppress obsolete error updates. Existing TUI code already guards draft and attachment clearing this way (plugins/chat/tui.ts:633-641). Preserve request IDs; do not retry an admitted send merely because its component was replaced. + +## Other exercised traces, not additional findings + +- Reporter deliver waits for a child, then accesses the plugin registry in the same old phase (subagents/tools.ts:136-153). After a registry migration that stale access intentionally faults; scheduler.ts:1360 processes failure before handover at :1366. Parent notification is lost, although the child's answer remains in its transcript. The migration needs a separate settlement checkpoint before versioned registry edits. This is an application phase-layout change, not a reason to undo stale protection. Introduce that layout while the registry schema remains compatible with already-running v1 reporters. +- Mixed facet versions are intentional. Example: models.worker setup fails and keeps G1, codemode.worker publishes G2. Compatible WorkerModels contracts continue; a G2-only contract fails that dependent clearly. Do not roll back unrelated facets. A rejected install report is still an install-attempt report for the reconnect routing gate; do not wait for an impossible complete-revision success. +- Shell admission must atomically combine deduplication and concurrency. Store outcome before transcript delivery; crash between Durable submission and outbox acknowledgement retries the same requestId. Crash before a receipt is deliberately lost/unknown and never blindly reruns an arbitrary command. A finishing/blocked job and pending deliveries remain activity. Already-discussed shell example problems are not counted again as findings. +- Two tabs with the same principal are two origin clients; auxiliary subscriptions are not extra viewers; a lingering hold keeps the worker alive but is excluded from Questions' live audience. Use origin IDs and the per-hold linger information rather than refs.length. +- Performance: retain whole-view source pointer/ops locally, bounded structurally-shared window remotely, paged history and bounded/throttled shell output. No token-by-token Chord JSON snapshot/clone. Models' initial file load must finish before resume; subsequent watching is routine work. + +## Cost and verdict + +Planning estimate, not measured patch size: + +| Plugin | Existing files changed | Lines materially touched | +| --- | ---: | ---: | +| chat | 5 | 550-850 | +| codemode | 3 | 100-180 | +| subagents | 6 | 300-450 | +| models | 8-10 | 600-900 | +| Selected plugins total | 22-24 | 1550-2380 | + +Allow another 6-10 p2 adapter/consumer files and 400-800 lines for host values, watch/source bridging, admission/exit checks, login's replacement for ModelConfig.resources, sessions' authoritative ChildStore reads, and consumer backoff. This excludes implementing Chord store/jobs/routines/wire, Loader D/SDK and their tests. The target's 600-900-line estimate is expressly documents-only; it is not a complete four-plugin migration estimate. + +Verdict: the core design holds. The missing persisted-policy and Chord-exit integration need explicit fixes; the rest needs nontrivial plugin migration, not another generic reload protocol. Do not present the migration as renaming cells/carriers and moving timers. + +## Verbatim essential migration sketches and recommendations + +Status annotation: proposed, uncompiled migration sketches, not implemented APIs or a functionality map. Helper names and new p2 adapter/registry names are proposals. The owner-decided mechanisms and the window-limit correction are recorded above. + +# Essential migration sketches + +Baseline: pi-private origin/private 6559589bd66bc909cfc1d694426799a3c4cd95f6. +Read-only review; these are sketches, not compilable patches. Names such as `bindReadableResource`, `watchConversation`, `loadAndCompose`, and `withCurrentWorker` below are proposed p2-local adapters, not additional Chord APIs. Existing parsing, rendering and validation helpers are retained. The cited revision spells standing work `self.routines`; a rename to `self.run` is mechanical. + +## Chat + +Keep transcript state in Durable. Do not copy the full ConversationView into a Chord document on each token. Preserve the existing 64-entry remote window, paged history, structural sharing, OutputBuffer and Progress throttling. The shared local conversation object, not each principal-specific remote facade, owns the shell job and persistent outbox. Give that object's type a stable storeKey. All authors still come from the stamped principal, never message text. + +```ts +// contract.ts: object identity includes session + conversation, not author/access/generation. +const ShellJob = defineJob("p2.chat.shell", { + version: 1, args: ShellArgs, progress: ShellProgress, result: ShellResult, + concurrency: 1, persist: true, +}); +const PendingShells = defineState("shell-deliveries", { + version: 1, schema: PendingShellsSchema, initial: () => ({ items: {} }), persist: true, +}); + +// worker.ts: shared local Conversation facade, storeKey declared on its type. +host.serve(Conversation, ({ session, conversation }, _principal, self) => { + const harness = host.value(HarnessKey); + const environments = host.value(EnvironmentsKey); + const liveness = host.value(LivenessKey); + const pending = self.state(PendingShells); + + // Native container/cache only. Its source adapter rebinds if the resource changes. + const bridge = self.resource("view", { + version: 1, open: () => makeAtomicViewBridge(null), close: (b) => b.close(), + }); + self.routines.loop("watch", async (ctx) => { + // A new Durable WatchHandle for each routine instance, never start an old handle twice. + const current = await harness.conversation(conversation, durableContext(ctx)); + if (!current) throw new Error("Unknown conversation"); + const watch = await current.watch(durableContext(ctx)); + try { + bridge.value.publish(watch.value); + watch.start(async (view, ops) => bridge.value.publish(view, ops)); + await waitUntilCancelledOrWatchEnds(watch, ctx); + } finally { + await watch.stop(); + } + }); + const wholeView = bindReadableResource(bridge); + const window = self.derive([wholeView], windowViewWithRetainedParts); + + const shells = self.job(ShellJob, { + run: async (args, ctx) => { + // Existing bounded output/attachment implementation; return actual native settlement. + // No transcript submission and no synthetic "cancelled = signal.aborted" here. + return executeShell(environments, args, ctx); + }, + finish: async (done, tx) => { + const q = await tx.draft(pending); + q.items[done.id] = shellDelivery(done); // JSON result/error, original author, stable requestId. + }, + }); + + self.routines.on(pending, async (queue, ctx) => { + // Count pending deliveries as work until acknowledgement, not just while this handler runs. + await liveness.call(async () => { + for (const [id, delivery] of Object.entries(queue.items)) { + const current = await harness.conversation(conversation, durableContext(ctx)); + if (!current) throw new Error("Unknown conversation"); + await current.submit(delivery.input, durableContext(ctx)); // requestId = shell:<job UUID>. + await self.tx(ctx, async (tx) => { + delete (await tx.draft(pending)).items[id]; + }); + } + }); + }); + + // Reconcile authoritative Durable agent configuration to sidebar/session metadata. + // Do not leave that update solely after a native configure await in a caller's old generation. + self.routines.on(agentConfigurationOf(wholeView), reconcileConversationMetadata); + return { + view: wholeView, window, liveShell: shells.running, + runShell: (args, ctx) => shells.start(args, ctx, { key: `${args.author.id}:${args.requestId}` }), + abortShell: ({ shellId }) => shells.abort(shellId), + ...existingConversationMethods(harness, conversation), + }; +}, { linger: 30_000 }); + +host.serve(Chat, (input, principal, self) => { + authorizeSessionRead(input, principal); + const conversation = self.get(Conversation, input); + if (!canSend(principal, input.session)) return viewer(conversation); + return participant(conversation, { + // Check authoritative persistent one-shot/write policy; absence of a registry is not permission. + beforeWrite: (ctx) => requireWritableConversation(input, ctx), + author: authorOf(principal), + gates: (ctx) => host.registry(InputGates).table(ctx), + }); +}); + +// web.tsx: same components, history retention and restored-draft claims. +export default defineFacet((_host, self) => self.register(Slots, "main", Main)); +async function send() { + const attempt = composer.beginAttempt({ conversation: identity, text: draft, files, replyTo }); + try { + await chat.send(await uploadAndExpand(attempt)); + composer.acknowledge(attempt); // compare identity + requestId + current draft/files; never clear newer edits. + } catch (error) { + composer.failIfCurrent(attempt, error); + } finally { + composer.endIfCurrent(attempt); + } +} + +// tui.ts / tui-commands.ts: register the existing surface/commands directly in the facet. +export default defineFacet((_host, self) => { + self.register(TuiSlots, ChatSurfaceKey, (runtime) => new ChatSurface(runtime)); + for (const [name, command] of Object.entries(CHAT_COMMANDS)) self.register(TuiCommands, name, command); +}); +``` + +No server facet. Drafts stay persisted by conversation + author, claim before acknowledgement, and are not keyed by the current viewer's principal access path. Retain Stop/withdraw/dequeue/edit/restore/compact/history semantics. Count active origin clients, not internal view subscriptions, for presence. A lingering client keeps its worker alive without being an audience member. Pending shell deliveries also need liveness while their routine is restarting/backing off: the implementation should register a count over the queue, rather than rely solely on the illustrative `liveness.call` above. + +## Codemode + +No server, web or TUI facet. Retain existing generic tool-card rendering. This remains a Durable extension, not a Chord job: scripts replay their journal after worker crashes. + +```ts +export default defineFacet((host, self) => { + const harness = host.value(HarnessKey); + const liveness = host.value(LivenessKey); + const settings = self.get(WorkerModels, {}); + const sandbox = self.resource("sandbox", { + version: 1, + open: () => new CodemodeSandbox({ memoryLimitBytes: CODEMODE_MEMORY_LIMIT_BYTES }), + close: (value) => value.close(), + }); + const calls = self.resource("calls", { + version: 1, + deps: [sandbox], // essential: cancel/settle calls before replacing their sandbox. + open: () => new Map<TaskId, ToolExecutionApi>(), + close: async (active) => { + if (liveness.stopping) return; // worker shutdown: Durable replays in the next worker. + await Promise.all([...active.values()].map(async (api) => { + api.output(`${REMOVED_MESSAGE}\n`); + await waitForRemovalOutputOrDeadline(harness, api, 2000); + await harness.abortTask(api.taskId, BACKGROUND_CONTEXT); + await harness.waitForTask(api.taskId, BACKGROUND_CONTEXT); + })); + }, + }); + const extension = createCodemodeExtension({ get sandbox() { return sandbox.value; } }); + const enabled = self.derive([settings.settings], (s) => s.codemode !== false); + self.register(ExtensionSwitches, extension.name, enabled); + self.register(Extensions, extension.name, { + ...extension, + tools: extension.tools.map((tool) => ({ + ...tool, + execute: async (args, api, context) => { + calls.value.set(api.taskId, api); + try { return await tool.execute(args, api, context); } + finally { calls.value.delete(api.taskId); } + }, + })), + }); +}); +``` + +The tracker is a data container; its behavior is fresh code. A compatible reload keeps the VM and calls. For a close-and-reopen replacement the calls dependent closes first, settles through Durable, then the sandbox closes. Do not treat a sandbox's spontaneous end as a completed Durable call. Preserve the removal output even if the output polling fails; production code retains the existing try/catch before mandatory abort. Removal policy should refuse while live tasks exist unless the owner chooses abort; this cleanup is also needed for a native version bump that does not remove the plugin. + +## Subagents + +The registry, anchors, reporter tasks, one-shot ownership, reported-answer IDs and inherited configuration remain transcript state in Durable. Only server sidebar metadata moves to Chord documents. + +```ts +const ChildRows = defineState("children-by-session", { + version: 1, keyed: true, persist: true, + schema: ChildRowSchema, initial: () => ({ generation: 0, children: [] }), +}); +const ChildrenIndex = defineState("children-index", { + version: 1, schema: ChildrenSchema, initial: () => ({}), // volatile live aggregate. +}); + +// server.ts: stable host-lifetime local ChildStore. +host.serve(ChildStore, (_input, _principal, self) => { + const hosts = host.value(SessionHostsKey); + const catalog = self.get(Catalog, {}); + const rows = self.state(ChildRows); + const children = self.state(ChildrenIndex); + const ids = self.derive([catalog.sessions], (list) => list.map((s) => s.id).sort()); + self.routines.on(ids, async (liveIds, ctx) => { + const recovered = new Map(await Promise.all(liveIds.map(async (id) => [id, await rows.get(id)]))); + await self.tx(ctx, async (tx) => { + const index = await tx.draft(children); + for (const id of Object.keys(index)) if (!liveIds.includes(id)) delete index[id]; + // A concurrent report already wrote the index: do not overwrite it with an older bootstrap read. + for (const id of liveIds) if (!(id in index) && recovered.get(id)) + index[id] = recovered.get(id).children; + }); + }); + return { + children, // UI cache, never authority for relocation or writes. + get: ({ session }) => rows.get(session), // await this when deciding relocation. + report: (report, ctx) => withCurrentWorker(hosts, report, ctx, () => self.tx(ctx, async (tx) => { + // Adapter holds the per-session worker lifecycle gate through this short transaction; + // merely sampling hosts.workers before an await would weaken today's generation fence. + const row = await tx.draft(rows, report.session, { tags: [SessionTag(report.session)] }); + row.generation = report.generation; + row.children = report.children; + (await tx.draft(children))[report.session] = report.children; + })), + }; +}, { lifetime: "host" }); +// The existing remote Subagents facade still stamps session/generation from its worker principal; +// owner sees all children, guest only its session. Never accept a caller-supplied worker identity. + +// worker.ts: declarations on the facet object, no contribute carrier. +export default defineFacet((host, self) => { + const harness = host.value(HarnessKey); + // Facet factories cannot self.get remote types; obtain this ref in the routine context. + const config = host.registry(HarnessConfig); + self.register(Extensions, "subagent-tools", createSubagentExtension({ + models: modelsForCurrentRequest(config), + })); + self.register(ReadOnlyConversations, "one-shot-agents", readOnlyFromDurableRegistry(harness)); + self.register(RelocationChecks, "own-cwd", ownCwdCheckFromDurableRegistry); + // Cwd followers move to the typed host-side registry/capability, not an opaque native export. + self.register(SessionCwdFollowerProviders, "subagents", subagentCwdFollowers); + self.routines.loop("report", async (ctx) => { + const feed = await ctx.get(Subagents, {}); + await reportChildrenFromFreshCommitSubscription(harness, feed, ctx); + }); + host.serve(SubagentControl, participantArchiveFacade(harness)); +}); + +// tools.ts: retain Durable tasks. Split the effect/settlement from versioned registry edits. +const Reporter = defineTask({ + name: "p2.subagent-reporter", version: 2, migrate: migrateReporterV1, + initial: () => ({ phase: "deliver" }), + phases: { + deliver: async (task, runtime, context) => { + const settled = await submitAndWaitWithExistingRequestId(task, runtime, context); + const result = await readImmutableAnswer(settled, runtime, context); + // No access to the plugin's migratable registry in this commit. + await runtime.commit(() => ({ status: "running", checkpoint: { phase: "record", ...result } }), context); + }, + record: (task, runtime, context) => runtime.commit(async (tx) => { + const registry = await tx.doc(SubagentsDoc, runtime.conversationId); + return deduplicateAnswerAndCheckpointReport(registry, task.state.checkpoint); + }, context), + report: submitToParentWithStableReporterRequestIdAndFinish, + }, + abort: terminalAborted, +}); + +// web.tsx: +export default defineFacet((_host, self) => + self.register(ToolRenderers, "subagent", SubagentRenderer)); +// tui.ts: +export default defineFacet((_host, self) => { + self.register(TuiToolRenderers, "subagent", subagentRenderers); + self.register(TuiSessionSources, "subagents", sessionSource); + for (const [name, command] of Object.entries(AGENT_COMMANDS)) self.register(TuiCommands, name, command); +}); +``` + +`SessionCwdFollowerProviders` is a proposed typed p2 registry replacing the existing opaque SessionCwdFollowers member; it must be present and checked before resume and relocation. Existing persisted one-shot/managed-cwd state makes the corresponding policy contributor required, even if this facet failed at first load. Metadata is never the authority for write permission. Archive still stops work before editing the Durable registry, refuses one-shot agents and keeps archived own-cwd agents relevant to relocation. The Reporter phase split is migration work: the deliberate stale-document failure otherwise terminally faults an old deliver phase after its child already answered. It does not justify changing the stale rule. Deploy this phase split without simultaneously changing the registry schema for already-running v1 reporters; they cannot be retroactively repaired. + +## Models + +There is no models web facet in this revision; chat imports the shared ModelPicker web UI. Keep that UI and its owner-only model access. The existing models TUI service and commands remain, using a typed surface key instead of a cross-generation instanceof test. + +```ts +// Native pod contains file handles, validated JSON, immutable current SDK model generation, +// counters and a host/base-defined observable. No Resources instance, plugin Proxy trap, +// ReloadObserver callback or plugin-defined class instance is retained across code reloads. +function declareModelResources(host, self) { + const pod = self.resource("files", { + version: 1, + open: ({ signal }) => openInitializedNativeModelPod(host.value(ModelResourceOptionsKey), signal), + close: (p) => closeFilesAfterTrackedWrites(p), + }); + self.routines.loop("config", async (ctx) => { + // Force fresh composition once for EVERY code generation, even when fingerprints match. + await loadAndCompose(pod, ctx, { forceCodeRefresh: true }); + const observer = watchModelFiles(host.value(ModelResourceOptionsKey)); + try { + while (!ctx.signal.aborted) { + await observer.next(ctx.signal); + await loadAndCompose(pod, ctx, { forceCodeRefresh: false }); + } + } finally { await observer.close(); } + }); + return pod; +} + +// server.ts: +export default defineFacet((host, self) => { + const pod = declareModelResources(host, self); + const config = bindReadableResource(pod); // one atomic snapshot; source rebinding is explicit. + host.serve(ModelConfig, (_input, _principal, object) => ({ + summary: object.derive([config], summarize), + snapshot: object.derive([config], publicResourceSnapshot), + setDefault: (choice, ctx) => saveValidatedDefault(pod, choice, ctx), + setEnabledModels: ({ patterns }, ctx) => saveEnabledPatterns(pod, patterns, ctx), + // Login must use methods on this owner; no Resources native exported to login. + ...modelAuthenticationMethods(pod), + })); + host.serve(Models, (_input, principal, object) => { + requireOwner(principal); + const local = object.get(ModelConfig, {}); + return existingOwnerModelsFacade(object, local); + }); +}); + +// worker.ts: +export default defineFacet((host, self) => { + const pod = declareModelResources(host, self); + const config = bindReadableResource(pod); + // A new proxy/facade in each setup; invocation lookup binds an immutable SDK generation. + // Never switch getAuth/stream implementations halfway through a waiting request. + const models = currentModelsFacade(pod); + self.register(HarnessConfig, "default", { + models: () => models, + settings: () => toHarnessSettings(pod.value.validatedSettings), + }); + host.serve(WorkerModels, (_input, _principal, object) => ({ + settings: object.derive([config], (value) => value.settings), + }), { lifetime: "host" }); +}); + +// tui.ts: +export default defineFacet((_host, self) => { + self.register(TuiSlots, ModelStateKey, (runtime) => new ModelState(runtime)); + for (const [name, command] of Object.entries(MODEL_COMMANDS)) self.register(TuiCommands, name, command); +}); +const stateOf = (ctx) => ctx.surface(ModelStateKey); // typed SDK token, not instanceof. +``` + +Move the initial model/config load into the host's required-start readiness, not merely a background loop that races resume(). Later failed files retain last-good validated config and publish bounded errors. Keep credentials private and file-backed, atomic file writes, in-flight-write tracking, fake mode, cached catalogs and no-network restoration. Force recompilation must also work with a currently invalid file by retaining last-valid parsed JSON as data. Preserve generation binding at invocation lookup. Do not retain plugin callbacks in a mutable SDK object just because the surrounding resource's data shape did not change. + +Cross-plugin consumers outside the four also change: notably login/server.ts and sessions/server.ts consume ModelConfig, and login currently reaches straight into config.resources. Sessions must also await authoritative ChildStore.get(session), rather than use the bootstrapped UI children map for relocation checks. That native export cannot simply disappear without replacing those operations with the typed methods shown above. + +## Verbatim historical reproduction source + +Status annotation: obsolete drain-prototype test API; execution outcomes are not retained. Current applicability and superseded expectations are classified above. Do not adopt these assertions as current target requirements. + +```ts +import assert from 'node:assert/strict'; +import { Type } from 'typebox'; +import { it } from 'vitest'; +import { createHost, local, method, type Self, type Facet, type EffectApi, defineRegistry, type Ctx } from '/home/badlogic/chord-handover/packages/chord/src/old/index.ts'; +const tick = () => new Promise<void>(r=>setImmediate(r)); +const sleep = (ms:number) => new Promise<void>(r=>setTimeout(r,ms)); +function latch(){let open=()=>{};const promise=new Promise<void>(r=>open=r);return {open,promise};} +const Probe = local('review.probe', {version:1,input:Type.Object({}),members:{call:method(Type.String(),Type.Unknown())}}); +function gen(build:(self:Self)=>(arg:string)=>unknown):Facet{return {name:'p',setup(api){api.serve(Probe,(_i,_p,self)=>({call:build(self)}));}};} + +it('a second reload must not start an effect before the first cleanup settled', async()=>{ + const done=latch();const started:number[]=[];const errors:unknown[]=[]; + const host=createHost({self:{kind:'host'},onError:e=>errors.push(e)}); + const version=(v:number)=>gen(self=>{self.effect('e',()=>{started.push(v);return v===1?()=>done.promise:()=>{};});return ()=>v;}); + try { + await host.install(version(1));await host.get(Probe,{}); + const first=await host.reload([version(2)],{drainDeadline:5,stallGrace:5}); + assert.equal(first.status,'installed_pending'); + const second=await host.reload([version(3)],{drainDeadline:5,stallGrace:5}); + assert.deepEqual(started,[1], `second report ${second.status}, stalled=${host.status()[0]?.stalled}`); + } finally {done.open();await tick();await host.close();} +}); + +it('queued raw run closures admitted before publication retain resource ownership',async()=>{ + const go=latch(); const closed:number[]=[];const seen:number[]=[];const errors:unknown[]=[];let emit:EffectApi['run']=()=>{};let queuedUse=()=>{}; + const host=createHost({self:{kind:'host'},onError:e=>errors.push(e)}); + const version=(v:number)=>gen(self=>{const conn=self.resource('conn',{version:v,open:()=>v,close:x=>{closed.push(x)}}); + self.effect('e',({run})=>{if(v===1){emit=run;queuedUse=()=>seen.push(conn.value);}});return ()=>conn.value;}); + try{ + await host.install(version(1));await host.get(Probe,{}); + emit(async()=>{await go.promise;seen.push(1);}); + emit(queuedUse); + await tick();await host.reload([version(2)]);go.open(); + for(let i=0;i<8;i++)await tick(); + assert.deepEqual(seen,[1,1], String(errors)); + }finally{go.open();await host.close();} +}); + +it('an activation blocked in old open must not prevent publishing new code',async()=>{ + const go=latch();let published=false;const host=createHost({self:{kind:'host'},onError:()=>{}}); + const old=gen(self=>{self.resource('r',{version:1,open:async()=>{await go.promise;return 1;}});return ()=>1;}); + try{ + await host.install(old);const ref=host.hold(Probe,{});await tick(); + const reload=host.reload([gen(()=>()=>2)],{onPublished:()=>{published=true}}); + await sleep(30);const before=published; + go.open();await reload;await ref.ready; + assert.equal(before,true,'old activation blocks publication'); + }finally{go.open();await host.close();} +}); + +it('retry does not rerun an added cell init that already succeeded',async()=>{ + const host=createHost({self:{kind:'host'},onError:()=>{}});let addedInits=0; + const version=(v:number,broken:boolean)=>gen(self=>{ + self.state('existing',Type.Number(), v===1?1:{init:()=>{if(broken)throw Error('broken');return 2;}},{version:v}); + if(v===2)self.state('added',Type.Number(),{init:()=>{addedInits++;return 3;}}); + return ()=>v; + }); + try{ + await host.install(version(1,false));await host.get(Probe,{}); + assert.equal((await host.reload([version(2,true)])).status,'installed_failed'); + assert.equal(addedInits,1); + assert.equal((await host.reload([version(2,false)])).status,'installed'); + assert.equal(addedInits,1); + }finally{await host.close();} +}); + + +it('a changed surviving consumer must still prohibit removal of its provider',async()=>{ + const Other=local('review.consumer',{version:1,input:Type.Object({}),members:{call:method(Type.String(),Type.Unknown())}}); + const host=createHost({self:{kind:'host'},onError:()=>{}}); + const consumer=():Facet=>({name:'c',setup(api){api.serve(Other,(_i,_p,self)=>{const ref=self.get(Probe,{});return {call:(arg,ctx)=>ref.call(arg,ctx)};});}}); + try{ + await host.install(gen(()=>()=>1));await host.install(consumer());const ref=await host.get(Other,{}); + await assert.rejects(host.reload([consumer()],{remove:['p']})); + assert.equal(await ref.call('read'),1); + }finally{await host.close();} +}); + +it('registry handlers own resources while they run and delay an exclusive handover',async()=>{ + const Routes=defineRegistry<(ctx:Ctx)=>Promise<number>>('review.routes'); + const go=latch();const closed:number[]=[];const host=createHost({self:{kind:'host'},onError:()=>{}}); + const version=(v:number):Facet=>({name:'p',setup(api){api.serve(Probe,(_i,_p,self)=>{ + const conn=self.resource('port',{version:v,mode:'exclusive',open:()=>v,close:x=>{closed.push(x)}}); + self.register(Routes,{route:async()=>{await go.promise;return conn.value;}});return {call:()=>v}; + },{lifetime:'host'});}}); + try{ + await host.install(version(1));const route=host.registry(Routes).get('route')!; + const pending=host.root(ctx=>route(ctx));pending.catch(()=>{});await tick(); + const reload=host.reload([version(2)]);await tick(); + const closedDuringHandler=[...closed];go.open();await Promise.allSettled([pending,reload]); + assert.deepEqual(closedDuringHandler,[], 'old registry handler is still using port 1'); + assert.equal(await pending,1); + }finally{go.open();await host.close();} +}); + +it('a new host lifetime object with a failing open is prepared before publication',async()=>{ + const host=createHost({self:{kind:'host'},onError:()=>{}});let published=false; + const added:Facet={name:'p',setup(api){api.serve(Probe,(_i,_p,self)=>{ + self.resource('broken',{version:1,open:()=>{throw Error('broken')}});return {call:()=>1}; + },{lifetime:'host'});}}; + try{ + await assert.rejects(host.reload([added],{onPublished:()=>{published=true}})); + assert.equal(published,false); + }finally{await host.close();} +}); + +``` diff --git a/apps/p2/docs/p3-reviews/session-data-scenarios.md b/apps/p2/docs/p3-reviews/session-data-scenarios.md new file mode 100644 index 000000000..bf436e6da --- /dev/null +++ b/apps/p2/docs/p3-reviews/session-data-scenarios.md @@ -0,0 +1,306 @@ +# Session data: write, export, import, continue (scenarios, for discussion) + +Status: discussion (2026-10-21), nothing decided. Assumes the open proposals P1 (a session is self-contained), P3 +(persisted documents only in input-less `local` objects) and shared storage (Chord's persisted session documents and +Durable commit into one storage, one seq). + +## The session directory + +``` +<data>/sessions/S1/ + session.sqlite class 1, log state: Durable (entries, tasks, documents) + Chord persisted documents + blobs/sha256/9f/9f3a… class 2, immutable content, named by its hash (artifact versions, uploads, sent files) + derived/artifacts/ class 3, a plugin's own files: index.sqlite (rebuildable, never exported or synced) + derived/search/ +``` + +Not in the directory: per-instance data (shares, devices, settings, read marks) in the server's storage; the catalog +on the server is a mirror of each session's metadata document. + +## The example plugin: artifacts with a full-text index + +```ts +const Artifact = defineState("artifact", { + schema: obj({ title: str(), type: str(), versions: arr(obj({ blob: str(), size: int(), at: int() })) }), + version: 1, keyed: true, persist: true, // key = artifact name + initial: () => ({ title: "", type: "text", versions: [] }), + checkpointWhen: () => true, +}); + +host.serve(ArtifactStore, (_, self) => { // local, input-less (P3) + const artifacts = self.state(Artifact); + const blobs = self.get(SessionBlobs, {}); // core: the session's blob store + const info = self.get(SessionInfo, {}); + const index = self.open(async () => openIndex(join(info.init.value.dir, "derived/artifacts/index.sqlite")), + (db) => db.close()); + + self.routines.loop("catch-up", async (ctx) => { // derived data repairs itself + for await (const name of artifacts.keys()) { + const a = await artifacts.get(name); + if (a && index.value.versions(name) !== a.versions.length) + index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx)); + } + }); + + return { + save: async ({ name, title, type, content }, ctx) => { + const bytes = utf8(content); + const blob = await blobs.put(bytes, ctx); // 1. content first + await self.tx(ctx, async (tx) => { // 2. the document referencing it + const a = await tx.draft(artifacts, name); + a.title = title; a.type = type; + a.versions.push({ blob, size: bytes.length, at: Date.now() }); + }); + const version = (await artifacts.get(name))!.versions.length; + index.value.put(name, version, content); // 3. derived, best effort + return { version }; + }, + search: ({ query }) => index.value.search(query), + }; +}); + +// the tool (Durable): the harness writes the tool result entry after it returns +execute: (args, api, context) => chord(context, async (ctx) => { + const { version } = await (await ctx.get(ArtifactStore, {})).save(args, ctx); + return { content: [{ type: "text", text: `${args.name} v${version} saved` }] }; +}), +``` + +## Trace 1: the agent saves an artifact (laptop) + +``` +t1 blobs.put blobs/sha256/9f/9f3a… (tmp, fsync, rename) not in the log +t2 Chord commit seq 41: document.change artifact "chart": versions[2] = { blob: "sha256:9f3a…" } +t3 index.put derived/artifacts/index.sqlite not in the log +t4 Durable commit seq 42: entry toolResult "chart v3 saved" +``` + +Crashes: after t1, an unreferenced blob (costs bytes, harmless); after t2, the index lags and the catch-up routine +repairs it at the next activation; after t3, Durable's recovery handles the tool call as today. + +## Trace 2: export + +``` +p3 export S1 → S1.p3session (one archive) + manifest.json { format: 1, session: "S1", head: 42 } + log.jsonl storage.export(): every record, Durable and Chord, in seq order, never filtered by plugins + blobs/… every file in blobs/ (no reference tracking; orphans from crashes come along) + (derived/ is skipped) +``` + +Consistent without stopping the session: the log export reads one storage snapshot at head 42; every blob a record up +to 42 references was written before that record (t1 before t2), so it is already in the directory. + +## Trace 3: import on another p3 instance + +``` +p3 import S1.p3session + 1. S1 exists here? refuse (or import under a new id: it is then another session) + 2. copy blobs/ into sessions/S1/blobs/, check each hash + 3. replay log.jsonl into a fresh sessions/S1/session.sqlite + 4. the server mirrors the session metadata document into its catalog + 5. first open: ArtifactStore activates, index.sqlite is missing, catch-up rebuilds it from documents + blobs +``` + +The artifacts plugin not installed there: its documents and blobs stay untouched; the index is built once it is. + +## Trace 4: continue on another instance (only with sync) + +``` +laptop (driver) outbox: [blob 9f3a…] [batch seq 41-42] blobs present at batch cut go first +desktop fetches batches, replays; fetches blobs by hash (eager or on read); builds its own index +takeover desktop asks for the next epoch, becomes driver; the laptop's copy turns read-only +``` + +## "A plugin uses its own SQLite for performance": two cases + +**Derived (an index, a cache, a materialized view):** as above. Writes to it are free; only the source goes through +the log. Export skips it, import and sync rebuild it. + +**Source of truth (data that exists nowhere else),** e.g. a plugin recording 1M terminal output lines in its own +database. Options: + +- **A, not allowed:** the source goes into documents, entries or blobs. Bulk append data is written as sealed chunks: + every N rows become one immutable blob, a document lists the chunk hashes, the open tail lives in a document. The + plugin's SQLite is again a derived query index over the chunks. Export, import and sync work unchanged. +- **B, plugin directories exported as opaque files:** export must stop the session's writers (a live SQLite file + copy is not consistent, and the database must agree with the log at head 42), then copy `derived/` along. Works for + export and import. Fails for sync: a changing database cannot ship incrementally (that is page shipping), so another + instance gets it only as a whole file at a handover. +- **C, plugin export hooks** (serialize to records, apply on import): for sync each plugin would also have to produce + incremental changes, i.e. its own log next to class 1. + +## The driver rule + +It bites only with sync. In p3 clients never write storage: the phone's calls go to the worker on the instance that +runs the session, which is the only writer anyway. Two instances holding S1 exist only through sync (then one drives) +or through import (then the import is another session). + +## Option D (2026-10-21, owner's concern: docs and blobs may not model everything) + +Queries are not limited by documents plus blobs (a derived SQLite may have any schema and index); the write model is. +Example: a file-history plugin, 10k rows `(path, before, after, entry, at)`; "directory a/ moved to b/" is one SQL +`UPDATE`, but 10k document changes. D: a plugin record kind in the session log; the plugin's own database is a +projection of its records (event sourcing; documents are the same idea with a fixed apply into memory). + +```ts +const FileHistory = defineLog("file-history", { + version: 1, + record: union([ + obj({ kind: lit("edit"), path: str(), before: str(), after: str(), entry: int(), at: int() }), + obj({ kind: lit("move"), from: str(), to: str(), at: int() }), + ]), +}); + +host.serve(FileHistoryStore, (_, self) => { + const log = self.log(FileHistory, { + open: (dir) => openDb(join(dir, "history.sqlite")), // any schema, any index + apply: (db, record) => { // deterministic: no clock, no I/O but db + if (record.kind === "edit") db.run("INSERT INTO edits VALUES (?,?,?,?,?)", …); + else db.run("UPDATE edits SET path = ? || substr(path, ?) WHERE path GLOB ?", …); + }, + }); + return { + move: ({ from, to }, ctx) => log.append({ kind: "move", from, to, at: Date.now() }, ctx), + versions: ({ path }) => log.db.all("SELECT * FROM edits WHERE path = ? ORDER BY at", path), + }; +}); +``` + +``` +append commit seq 57: { log: "file-history", record: move a/ → b/ } (same storage, may share a tx with documents) + then apply + "_applied = 57" in one SQLite transaction of history.sqlite +crash between the two: on open Chord applies records after _applied +export records are log records: already in log.jsonl; history.sqlite skipped +import replay the log; first open: empty db, apply every record from the start (or from a snapshot) +sync records ship in batches like everything else; the other instance applies them +compact log.snapshot(): VACUUM INTO → blob, record { snapshot: hash, seq }; records before it may be dropped; + import starts from the snapshot blob and applies the rest +``` + +Rules and costs: `apply` deterministic and pure over its database; every record version is applied forever (or until a +snapshot); one writer per log (the object's line); a snapshot is a SQLite file (a Durable Object cannot load one, it +replays records instead). + +## Option E (2026-10-21, owner: an app-layer abstraction; Chord provides primitives) + +Owner: blobs plus documents recording them are already app layer; the app knows what to sync; plugins get a storage +location in the session directory that is synced too; open issue: consistency on sync. + +The problem, traced (two stores, no common order): + +``` +driver t1 history.sqlite: row { path: "x", entry: 812 } + t2 session.sqlite seq 42: entry 812 +replica has the SQLite changes up to t1, the log only up to seq 41: the row points to a missing entry +takeover at that moment: the new driver writes another entry 812 at seq 42: the row points to the wrong entry +``` + +Mechanism: one position sequence across all stores of a session (app layer). Every store, in the same transaction as +its change, takes the next position and records the change in its own outbox (SQLite: an outbox table in the same +database; JSONL: the appended lines themselves). A batch is cut at position P once nothing at or below P is still +committing; it holds every store's records up to P. A receiver applies them in position order, so it is always at a +state the driver had. After a crash the counter restarts at the maximum over all stores. + +| store kind | change record | apply on a replica | snapshot (export, compaction) | +| --- | --- | --- | --- | +| Chord / Durable storage | the commit's writes | `replay` | `export()` | +| blob directory | "added hash" plus the bytes | write the file | the files | +| append-only JSONL | the appended lines | append | the file | +| SQLite (app abstraction) | changeset from SQLite's session extension, stored in the same transaction | `applyChangeset` | `VACUUM INTO` | +| files changed in place | none | none | not syncable: derived only | + +Checked on Node 24 (`node:sqlite`): `createSession({ table })`, `changeset()` before `COMMIT`, insert into an outbox +table in the same transaction, `applyChangeset` on a second database gives the same rows. Limits: changesets are per +row (a 10k-row move is 10k row changes, tens of bytes each, no pages); schema changes are not in changesets, so plugin +migrations are versioned and run on every replica before changes of the new version; Cloudflare's Durable Object SQLite +API may not expose the session extension (to check). + +Chord's part: a storage that can be wrapped (every commit's writes, in order, with a hook inside the commit to take a +position and write the outbox record) plus `export()`, `replay()` and `head()`. Consequence: ordering no longer needs +Chord documents in Durable's storage; shared storage would only save a file. + +## Generalized session SQLite store (owner, 2026-10-21, proposal) and outbox cost + +Owner: one generalized store implementing the outbox mechanics; plugins give it a schema plus versions/migrations +and get a SQLite interface ("SQLite all the things"). Prototype of the mechanics: `session-sync-prototype/`. + +Measured on slayer (Node 24.4.1, WAL, `synchronous=NORMAL`), table `edits` with 2 KB before/after text per row, +one transaction per operation: + +| workload | plain | changeset outbox | patchset outbox | +| --- | --- | --- | --- | +| 10k inserts | 244 ms | 584 ms, outbox 40.5 MB (= row data 40.2 MB) | 568 ms, 40.5 MB | +| rename 10k rows (one UPDATE) | 137 ms | 198 ms, 418 KB | 191 ms, 259 KB | +| 1k one-column updates | 11 ms | 39 ms, 49 KB | 38 ms, 36 KB | + +Per transaction about +30 µs. An insert's record is the row (the same holds for any replication log, including a +Durable outbox: an entry's record is the entry). Set-based updates cost 26-42 bytes per changed row. The outbox is +trimmed after upload, so disk holds only the unsynced backlog. + +## Recording SQL statements, verified by an effect hash (2026-10-21, proposal) + +Record each write statement with its parameters instead of the changed rows; check determinism instead of trusting +it. Checked on Node 24.4.1: + +- Nondeterministic built-ins can be blocked by overriding them per connection with `db.function(name, { varargs: + true }, throwing)`: `random()`, `randomblob()`, `datetime('now')`, `CURRENT_TIMESTAMP`, `unixepoch()` all throw; + `abs()` still works. (`setAuthorizer` is not in Node 24.4.) +- SQLite's patchset of a statement is byte-identical on two databases with the same rows even when their physical + insertion order differs (5,000 rows inserted ascending vs descending: same 118,904-byte patchset, same SHA-256), and + differs after one replica lost a row. So a 16-byte hash of the patchset, recorded with the statement, detects any + divergence at the exact statement on the replica. +- Record size of the 5,000-row rename: the statement is 69 bytes (plus a 16-byte hash) instead of a 119 KB + patchset. + +Shape: the plugin writes through `tx.run(sql, ...params)`; the store runs it inside a SQLite session, records `{ pos, +statements: [[sql, params]], hash(patchset) }` in `_outbox` in the same transaction, and discards the patchset. The +replica runs the statements inside a session and compares the hash; a mismatch stops that store and restores it from +a snapshot (and reports the plugin's nondeterministic SQL). Costs: the session extension still runs on both sides +(CPU, about +30 µs per transaction), no extra disk. Remaining sources of divergence that the hash catches but nothing +prevents: `INSERT ... SELECT` or `LIMIT` without a total `ORDER BY`, different SQLite versions across machines. + +## Statements Durable's and Chord's SQLite storages use (inventory, 2026-10-21) + +Sources: `packages/durable/src/storage/sqlite/{storage,migrations}.ts`, `packages/chord/src/storage/sqlite/{storage,schema}.ts`. + +Reads: + +| Shape | Durable | Chord (frozen next/) | +| --- | --- | --- | +| get by primary key | conversations, entries (+ `commit_seq`), tasks, submissions, documents, record_ids, metadata singleton | documents, ids, metadata | +| equality filters + id range + order + limit (paging) | conversations by owner; tasks by any of conversation, kind, status, abortRequested, background; submissions by conversation, status; entries by conversation with `id >= ? AND id <= ?`, asc or desc | keys of (owner, definition) after a cursor, ordered by `key_order` | +| latest matching row (`ORDER BY … DESC LIMIT 1`) | entry heads (`head IS NOT NULL`, partial index); document revisions: newest base `seq <= ?`, newest revision | newest base | +| range of a composite key | revision tail `document_id = ? AND seq > ? AND seq <= ? ORDER BY seq` | same | +| lookup by secondary key | submission by `(conversation_id, request_id)`; current document by address (5 columns) `AND retired_at IS NULL` | current document by address (4 columns, unique partial index) | +| point in time | documents `created_at <= ? AND (retired_at IS NULL OR retired_at > ?)` | none | +| tags | none | `byTag` (dropped by proposal P3) | + +Writes: insert row; upsert whole row on primary-key conflict (tasks, submissions); insert-or-ignore (record_ids, tags); +update columns by primary key (metadata counters, retiring a document); delete by key prefix (all revisions of a +document; tags of a document). Every commit is one transaction; multi-query reads (document materialization) run in a +transaction for one consistent state. Migrations: DDL, including a table rebuild to change a CHECK constraint. + +Not used anywhere: joins, aggregates (not even `count`), `GROUP BY`, `LIKE`, subqueries, expressions in `SET` (only +values), set-based updates other than delete-by-prefix. Details a backend must match: string order equal to +JavaScript UTF-16 code-unit order (Chord's `key_order` encoding exists for this), integers beyond 2^53 stored as text +(`next_id`), partial and unique indexes. p2's plugins additionally use `count`, `IN (subquery)`, joins, `GROUP BY` with +`max`, and `SET x = x + 1` / `max(x, ?)` (catalog and sharing, server-side). + +## Generic JSON rows plus local SQL views (2026-10-21, proposal after "IndexedDB may not be enough") + +The store keeps every table's rows in one generic layout, `rows(tbl, key, v, row JSON)`, so it needs no plugin code; +the plugin, where its code runs, creates local views and expression indexes (derived, never synced) and reads with full +SQL. Measured on Node 24.4.1, 100k `file-history/edits` rows plus 20k `turns` rows: + +| query | time | plan | +| --- | --- | --- | +| versions of a file (`WHERE path = ? ORDER BY at DESC LIMIT 20`) | 0.07 ms | covering expression index | +| edits per model (join + `GROUP BY`) | 129 ms | index on the join key, temp B-tree for the group | +| most edited files (`GROUP BY path ORDER BY n DESC LIMIT 5`) | 49 ms | scan of the table's rows | +| big edits in a time window (two ranges, no index on `at`) | 30 ms | scan of the table's rows | + +Write guard without `setAuthorizer` (Node 24.4.1): temporary triggers `BEFORE INSERT/UPDATE/DELETE ON main.rows WHEN +store_writing() = 0 ... RAISE(ABORT)` with `store_writing` a JS function the store flips around its own writes; plain +`INSERT`, `UPDATE`, `DELETE` and `WITH ... DELETE` from plugin SQL are refused. DDL (`DROP TABLE rows`) is not stopped +by triggers. diff --git a/apps/p2/docs/p3-reviews/session-sync-explainer/book.md b/apps/p2/docs/p3-reviews/session-sync-explainer/book.md new file mode 100644 index 000000000..a888f9da4 --- /dev/null +++ b/apps/p2/docs/p3-reviews/session-sync-explainer/book.md @@ -0,0 +1,356 @@ +--- +title: One Session, Many Stores: exporting and syncing a p3 session +cover: Many / Stores +kicker: A discussion book · p3 session data · 2026-10-21 +subtitle: How a session whose plugins keep their own SQLite files can be exported, imported and continued on another machine without coming apart +stats: + - 4 | kinds of storage in one session directory + - 1 | position counter per session, shared by all stores + - 54 B | changeset of a two-row SQL UPDATE (measured, Node 24) + - 0 | Chord changes beyond a wrappable storage +note: | + A proposal for discussion, not a decision. Written for the owner after the discussion of 2026-10-21 (option E in + `apps/p2/docs/p3-reviews/session-data-scenarios.md`). Assumes the open proposals P1 (a session is self-contained) + and P3 (persisted documents only in input-less `local` objects). The SQLite changeset behaviour was checked on + slayer with Node 24.4.1 (`node:sqlite`); everything else is design. Notes marked *review* are open points. +colophon: One Session, Many Stores · first edition · 2026-10-21 · based on session-data-scenarios.md and the redesign log +--- + +# Before we begin {.preface} + +Mario works in session S1 on his laptop. The agent edits files, and a plugin called **file-history** remembers every +edit: which file, which entry of the conversation made it, the content before and after. It keeps those rows in its +own SQLite database, `history.sqlite`, because it needs SQL: "every version of `src/app.ts`", "every file the last +turn touched", "the directory `a/` was renamed to `b/`, update ten thousand rows". + +In the evening Mario wants to continue S1 on his desktop. Or he exports S1 to a file and imports it on another p3 +instance. Either way the desktop must end up with the conversation (Durable), the plugins' documents (Chord), the +artifact contents (blobs) and `history.sqlite`, and all four must agree with each other. A row in `history.sqlite` +says "entry 812 made this edit"; entry 812 must exist on the desktop and be the same entry. + +Copying files does not do that. A live SQLite file copied while it is written is not a consistent database, and +`session.sqlite` and `history.sqlite` copied a moment apart do not agree with each other. Shipping only the +conversation and rebuilding the rest works for indexes, not for `history.sqlite`: its rows exist nowhere else. + +## Seven things to know + +:::findings +1. **A session is a directory of stores.** The log (`session.sqlite`), blobs, plugin databases and derived caches all + live in `sessions/S1/`. +2. **Immutable and derived data are easy.** Blobs are written before the document that names them; derived data is + rebuilt. Export and import need nothing else for them. +3. **Mutable plugin stores are the problem.** Two stores changing side by side can arrive on another machine at + different points in time, and then they disagree. +4. **One position counter fixes the order.** Every change in every store of the session takes the next position from + one counter. Positions say what happened before what, across stores. +5. **Each store records its own changes in its own transaction.** The change and its outbox record commit together, + so a crash can never lose one without the other. +6. **SQLite can record its own changes.** Its session extension turns any SQL write into a compact changeset of rows, + which another database applies. A plugin writes plain SQL. +7. **Chord only provides a wrappable storage.** The counter, the outboxes, the batches and the store kinds are app + layer (p3's sync layer). +::: + +## Words used throughout + +A **store** is one place where a session keeps data: Durable's and Chord's storage (`session.sqlite`), the blob +directory, a plugin's SQLite database, an append-only JSONL file. A **change** is one committed transaction of one +store. A **position** is a number from the session's counter that every change takes when it commits. An **outbox** +is where a store records its changes with their positions, inside the same transaction as the change. A **batch** is +all changes of all stores up to one position, shipped together. The **driver** is the instance that writes the +session; a **replica** is a copy on another instance that applies batches. + +# What a Session Holds {#holds lede="The four kinds of data in a session directory, and which of them are hard."} + +Everything of S1 lives under one directory on the instance that runs it. The server keeps only mirrors (the catalog) +and its own instance data (shares, devices, settings). + +:::flow wide | The session directory of S1. Accent marks the store that makes export and sync hard. +row "Log" blue +box log "session.sqlite" "Durable entries, tasks, documents; Chord documents" +row "Content" muted +box blobs "blobs/" "artifact versions, uploads; named by hash, never changed" +row "Plugin stores" accent +box history "history.sqlite" "file-history: rows exist nowhere else" accent +box index "search index" "derived from the log, rebuildable" dashed +note "The server holds only mirrors (catalog) and instance data (shares, devices, settings)." +::: + +| Kind | Example | Changes | Export and sync | +| --- | --- | --- | --- | +| log | conversation, documents | append and update | must ship, in order | +| blob | artifact v3 contents | never | ship the file | +| derived | full-text index | rebuilt | never ship, rebuild | +| plugin store | `history.sqlite` | update in place | the open problem | + +# The Easy Cases {#easy lede="Why blobs and derived data need no machinery, and what that teaches about the hard case."} + +The artifacts plugin saves a new version. It writes the content as a blob first, then commits a document that names +the blob's hash. The order is the whole trick. + +:::sequence animate | Saving artifact "chart" version 3. The blob exists before anything refers to it. +actor tool "artifact tool" +actor blobs "blobs/" muted +actor log "session.sqlite" blue +tool -> blobs: write sha256:9f3a… (tmp, fsync, rename) +tool => log: seq 41: chart.versions[2] = { blob: "sha256:9f3a…" } +tool => log: seq 42: entry 812 "chart v3 saved" +::: + +An export reads the log at some head, say 42, and copies every file in `blobs/`. Any blob a record up to 42 refers to +was written before that record, so it is in the directory. The session does not have to stop. An import replays the +log into a fresh `session.sqlite`, copies the blobs and checks their hashes. + +## Derived data rebuilds itself + +The artifacts plugin also keeps a full-text index over artifact contents, in its own SQLite file under +`derived/artifacts/`, because searching documents one by one is too slow. The index holds nothing of its own: every +row can be recomputed from the `artifact` documents and the blobs they name. So it is never exported or synced. A +routine compares the index with the documents whenever the object starts and fills in whatever is missing: + +```ts +const Artifact = defineState("artifact", { + schema: obj({ title: str(), type: str(), versions: arr(obj({ blob: str(), size: int(), at: int() })) }), + version: 1, keyed: true, persist: true, // key = artifact name + initial: () => ({ title: "", type: "text", versions: [] }), + checkpointWhen: () => true, +}); + +host.serve(ArtifactStore, (_, self) => { // local, input-less + const artifacts = self.state(Artifact); + const blobs = self.get(SessionBlobs, {}); // app layer: the session's blob store + const info = self.get(SessionInfo, {}); + const index = self.open(async () => openIndex(join(info.init.value.dir, "derived/artifacts/index.sqlite")), + (db) => db.close()); + + self.routines.loop("catch-up", async (ctx) => { // the index repairs itself + for await (const name of artifacts.keys()) { + const a = await artifacts.get(name); + if (a && index.value.versions(name) !== a.versions.length) + index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx)); + } + }); + + return { + save: async ({ name, title, type, content }, ctx) => { + const bytes = utf8(content); + const blob = await blobs.put(bytes, ctx); // 1. content first + await self.tx(ctx, async (tx) => { // 2. the document naming it + const a = await tx.draft(artifacts, name); + a.title = title; a.type = type; + a.versions.push({ blob, size: bytes.length, at: Date.now() }); + }); + const version = (await artifacts.get(name))!.versions.length; + index.value.put(name, version, content); // 3. derived, best effort + return { version }; + }, + search: ({ query }) => index.value.search(query), + }; +}); +``` + +The same routine covers three situations without any special code: + +:::steps | One routine, three situations ++++ Normal save +`save` writes the blob, commits the document, then updates the index. The routine finds nothing to do. ++++ Crash after the commit +The process dies between step 2 and step 3. The document says version 3, the index still has version 2. On the next +start the routine sees the difference, reads the blob of version 3 and indexes it. ++++ Import or sync +On the desktop `derived/artifacts/` does not exist. `openIndex` creates an empty index; the routine finds every +artifact missing and indexes them all from the documents and blobs that came with the session. +::: + +:::note | Cost of the catch-up +This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store would +keep the last position it applied and read only the changes after it (the outboxes of chapter V give exactly that). +::: + +Both cases work because nothing ever changes in place: a blob is final when written, and a derived index has no data +of its own. `history.sqlite` breaks both assumptions. Its rows change in place, and they are the only copy. + +# Two Stores, No Common Order {#problem lede="What goes wrong when history.sqlite and the log travel separately."} + +Suppose every store ships its own changes to the desktop as it pleases: `session.sqlite` in batches of commits, +`history.sqlite` as a stream of its own. The agent edits `x.ts`; the edit is recorded in both stores. + +:::steps | One edit, two stores, shipped separately ++++ On the laptop +The tool records the edit in `history.sqlite` (row: path `x.ts`, entry 812), then Durable commits entry 812 at seq +42. On the laptop both exist and agree. ++++ The history stream arrives first +The desktop has received the `history.sqlite` changes, including the row, but the log only up to seq 41. The row +says entry 812 made the edit; the desktop has no entry 812. ++++ Mario takes over now +The laptop's lid is closed; the desktop becomes the driver. Its agent writes the next entry, which gets id 812: a +different message. The row now points to the wrong entry, permanently. ++++ The opposite order +If the log arrives first instead, entry 812 says "x.ts edited" and the history has no row for it. Either way the +desktop holds a state the laptop never had. +::: + +The fault is not in either store. Each one is consistent on its own. What is missing is an order **across** stores: +which change of `history.sqlite` happened before which commit of the log. + +# One Counter for the Session {#counter lede="A single sequence of positions that every store takes from when it commits."} + +The session gets one counter, kept in memory by p3's sync layer on the driver. Every store, in the transaction that +commits a change, takes the next position and writes the change with that position into its own outbox. Positions +increase across all stores of the session, so they order everything. + +:::sequence animate | The same edit with positions. Each store writes its outbox inside its own transaction. +actor tool "edit tool" +actor counter "counter" "sync layer, memory" accent +actor history "history.sqlite" "plugin store" +actor log "session.sqlite" blue +tool -> history: BEGIN; INSERT row (x.ts, entry 812) +history -> counter: next() +counter --> history: 7 +history => history: outbox row (7, changeset); COMMIT +tool -> log: commit entry 812 +log -> counter: next() +counter --> log: 8 +log => log: seq 42 with position 8, same transaction +::: + +After the edit the two outboxes look like this: + +| Store | Position | Change | +| --- | --- | --- | +| `history.sqlite` | 7 | changeset: insert row (x.ts, entry 812) | +| `session.sqlite` | 8 | seq 42: entry 812 | + +The sync layer cuts a **batch** at a position P: every change of every store with a position up to P. A replica +applies a batch in position order, store by store as the positions say. After applying a batch it holds exactly the +state the driver had at P, for all stores together. The bad states of the previous chapter cannot happen: a replica +has position 8 only together with position 7. + +:::flow wide animate | Shipping. Batches are cut across stores; the replica applies them in position order. +row "Driver" accent +box ho "history outbox" "7, 11, 12" +box lo "log outbox" "8, 9, 10, 13" +box cut "cut at 12" "nothing ≤ 12 still committing" accent +row "Replica" blue +gap +box apply "apply in order" "history 7, log 8 9 10, history 11 12" +box batch "batch ≤ 12" "7 8 9 10 11 12" blue +arrow ho -> cut +arrow lo -> cut +arrow cut -> batch "upload" +arrow batch -> apply +note "Position 13 waits for the next batch, together with whatever else comes after 12." +::: + +**Cutting needs one rule.** The counter hands out position 12 to a store whose transaction has not committed yet, +while position 13 already committed elsewhere. A cut at 13 would miss 12. So the counter remembers the positions it +handed out that are not committed yet, and a cut goes only up to just below the smallest of them. A transaction that +fails gives its position back as a gap; gaps are fine, order is what counts. + +# Capturing a SQLite Change {#sqlite lede="How a plugin writes plain SQL and still produces a change record another database can apply."} + +A plugin store must turn its writes into change records. For JSONL that is the appended lines themselves. For SQLite +it would be a burden on every plugin, except that SQLite has the **session extension**: it watches tables and turns +whatever SQL ran into a **changeset**, a compact list of row inserts, updates and deletes. Another database applies +it. `node:sqlite` exposes it. This ran on slayer with Node 24.4.1: + +```js +import { DatabaseSync } from "node:sqlite"; +const laptop = new DatabaseSync(":memory:"), desktop = new DatabaseSync(":memory:"); +// both: CREATE TABLE edits(id INTEGER PRIMARY KEY, path TEXT); CREATE TABLE outbox(pos INTEGER PRIMARY KEY, cs BLOB) + +const session = laptop.createSession({ table: "edits" }); +laptop.exec("BEGIN"); +laptop.exec("UPDATE edits SET path = 'b/' || substr(path, 3) WHERE path GLOB 'a/*'"); // plain SQL +const cs = session.changeset(); // 54 bytes for two rows +laptop.prepare("INSERT INTO outbox VALUES (?, ?)").run(7, cs); // same transaction +laptop.exec("COMMIT"); + +desktop.applyChangeset(cs); // desktop now has b/x and b/y +``` + +So p3 could offer plugins a session SQLite as a store kind. The plugin opens it, writes SQL in transactions, and the +wrapper does the rest: start a session on the plugin's tables, take a position before `COMMIT`, write the changeset to +the outbox table, commit. The plugin never sees positions or outboxes. + +:::review | Limits of changesets +**Rows, not commands.** Renaming `a/` to `b/` over ten thousand rows is ten thousand row changes in the changeset +(tens of bytes each), not one command. Still logical rows, never database pages. +**No schema changes.** `CREATE TABLE` and `ALTER TABLE` are not in changesets. A plugin's schema migrations must be +versioned and run on every replica before changes made with the new schema are applied. +**Cloudflare.** The Durable Object SQLite API may not expose the session extension; not checked. +::: + +# Crashes {#crashes lede="Why the outbox lives inside each store, and how the counter restarts."} + +Why not one shared outbox file for all stores? Because the change and its record must commit together. With a shared +file, the plugin commits its rows, the process dies before the changeset reaches the shared file, and the changeset +is gone: it existed only in memory. The rows are on disk and will never be shipped. Each store therefore keeps its +outbox inside itself, in the same transaction as the change: an outbox table in a SQLite database, the record in +`session.sqlite`'s commit, the lines themselves in a JSONL file. + +:::steps | The laptop dies in the middle of a turn ++++ Before the crash +Positions 1 to 14 are handed out. 7, 11, 12 committed in `history.sqlite`; 8, 9, 10, 13 in `session.sqlite`; 14 was +still in a `history.sqlite` transaction. Batch ≤ 12 was uploaded. ++++ The crash +Position 14's transaction never committed, so neither its rows nor its outbox record exist. Nothing half-done is +left in any store. ++++ Restart +The sync layer reads the highest position in every outbox (12 and 13) and starts the counter at 14. Position 13 is +committed and not yet uploaded; it goes into the next batch. ++++ Nothing is lost or reordered +Every committed change has its record; every record has its change. The next batch continues at 13. +::: + +# What Each Store Needs, and What Chord Provides {#kinds lede="The store kinds p3 would offer plugins, and the small part that falls to Chord."} + +| Store kind | Change record | Applied on a replica by | Snapshot (export, compaction) | +| --- | --- | --- | --- | +| Durable/Chord storage | the commit's writes | `replay` | `export()` | +| blob directory | "added hash" plus the bytes | writing the file | the files | +| append-only JSONL | the appended lines | appending | the file | +| plugin SQLite | changeset, in the same transaction | `applyChangeset` | `VACUUM INTO` | +| files changed in place | none | none | cannot sync: derived data only | + +**Export and import** use the same pieces. An export takes a snapshot of every store at one cut position P (the +log's `export()`, `VACUUM INTO` for each plugin database, the blob files, the JSONL files) and writes them into one +archive with P in its manifest. An import restores each snapshot into a fresh session directory. Later changes, on a +replica, continue from P. + +**Chord's part** is small. Its storage (and Durable's) must be wrappable: the sync layer sees every commit's writes +in order and gets a hook inside the commit to take a position and write the outbox record in the same transaction. +And the storage needs `export()`, `replay()` and `head()`. The counter, outboxes, batches, store kinds and the +session directory are p3's. + +One consequence for an open question: ordering no longer needs Chord's documents to live in Durable's storage. If +they had their own file, it would be one more store with its own outbox, ordered by the same counter. Sharing the +storage would only save a file. + +# Open Questions {.epilogue lede="What this proposal needs decided before anything is built."} + +The proposal: a session is a directory of stores; p3's sync layer orders every change of every store through one +position counter; each store kind records its changes in its own transaction; plugins get store kinds (SQLite through +changesets, JSONL, blobs) in the session directory; files changed in place may hold only derived data. Chord provides +a wrappable storage with `export`, `replay` and `head`. + +## Needs a decision + +- **The direction.** Is this the model for p3 sessions? +- **Store kinds.** Which kinds p3 offers plugins at first (SQLite and blobs, or JSONL too). +- **Shared storage.** Whether Chord's persisted session documents still share Durable's storage, now that ordering + does not need it. + +## To verify + +- The session extension's cost on large tables and wide updates (a benchmark of the 10k-row rename). +- Whether Cloudflare's Durable Object SQLite exposes changesets; if not, what a plugin store means there. + +# Glossary {.appendix} + +**Batch:** all changes of all stores up to one position. **Blob:** a file named by the hash of its content, never +changed. **Changeset:** SQLite's record of the rows a transaction inserted, updated or deleted. **Cut:** choosing the +position a batch ends at; never past a position still committing. **Derived data:** data rebuilt from other stores, +never shipped. **Driver:** the instance that writes a session. **Outbox:** a store's own record of its changes with +positions, written in the same transaction. **Position:** a number from the session's counter, taken by every change. +**Replica:** a copy of a session on another instance that applies batches. **Store:** one place a session keeps data. diff --git a/apps/p2/docs/p3-reviews/session-sync-explainer/index.html b/apps/p2/docs/p3-reviews/session-sync-explainer/index.html new file mode 100644 index 000000000..c2bc29337 --- /dev/null +++ b/apps/p2/docs/p3-reviews/session-sync-explainer/index.html @@ -0,0 +1,505 @@ +<!doctype html> +<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> +<title>One Session, Many Stores: exporting and syncing a p3 session + + + + +
+
+
A discussion book · p3 session data · 2026-10-21
+

Many
Stores

+

How a session whose plugins keep their own SQLite files can be exported, imported and continued on another machine without coming apart

+
4kinds of storage in one session directory
1position counter per session, shared by all stores
54 Bchangeset of a two-row SQL UPDATE (measured, Node 24)
0Chord changes beyond a wrappable storage
+

A proposal for discussion, not a decision. Written for the owner after the discussion of 2026-10-21 (option E in +apps/p2/docs/p3-reviews/session-data-scenarios.md). Assumes the open proposals P1 (a session is self-contained) +and P3 (persisted documents only in input-less local objects). The SQLite changeset behaviour was checked on +slayer with Node 24.4.1 (node:sqlite); everything else is design. Notes marked review are open points.

+
+
+ + +
+

Before we begin

+

Mario works in session S1 on his laptop. The agent edits files, and a plugin called file-history remembers every +edit: which file, which entry of the conversation made it, the content before and after. It keeps those rows in its +own SQLite database, history.sqlite, because it needs SQL: "every version of src/app.ts", "every file the last +turn touched", "the directory a/ was renamed to b/, update ten thousand rows".

+

In the evening Mario wants to continue S1 on his desktop. Or he exports S1 to a file and imports it on another p3 +instance. Either way the desktop must end up with the conversation (Durable), the plugins' documents (Chord), the +artifact contents (blobs) and history.sqlite, and all four must agree with each other. A row in history.sqlite +says "entry 812 made this edit"; entry 812 must exist on the desktop and be the same entry.

+

Copying files does not do that. A live SQLite file copied while it is written is not a consistent database, and +session.sqlite and history.sqlite copied a moment apart do not agree with each other. Shipping only the +conversation and rebuilding the rest works for indexes, not for history.sqlite: its rows exist nowhere else.

+

Seven things to know

+
    +
  1. A session is a directory of stores. The log (session.sqlite), blobs, plugin databases and derived caches all +live in sessions/S1/.
    2. +
  2. Immutable and derived data are easy. Blobs are written before the document that names them; derived data is +rebuilt. Export and import need nothing else for them.
    3. +
  3. Mutable plugin stores are the problem. Two stores changing side by side can arrive on another machine at +different points in time, and then they disagree.
    4. +
  4. One position counter fixes the order. Every change in every store of the session takes the next position from +one counter. Positions say what happened before what, across stores.
    5. +
  5. Each store records its own changes in its own transaction. The change and its outbox record commit together, +so a crash can never lose one without the other.
    6. +
  6. SQLite can record its own changes. Its session extension turns any SQL write into a compact changeset of rows, +which another database applies. A plugin writes plain SQL.
    7. +
  7. Chord only provides a wrappable storage. The counter, the outboxes, the batches and the store kinds are app +layer (p3's sync layer).
    8. +
+

Words used throughout

+

A store is one place where a session keeps data: Durable's and Chord's storage (session.sqlite), the blob +directory, a plugin's SQLite database, an append-only JSONL file. A change is one committed transaction of one +store. A position is a number from the session's counter that every change takes when it commits. An outbox +is where a store records its changes with their positions, inside the same transaction as the change. A batch is +all changes of all stores up to one position, shipped together. The driver is the instance that writes the +session; a replica is a copy on another instance that applies batches.

+
+ +
I

What a Session Holds

The four kinds of data in a session directory, and which of them are hard.

+

Everything of S1 lives under one directory on the instance that runs it. The server keeps only mirrors (the catalog) +and its own instance data (shares, devices, settings).

+
LOGCONTENTPLUGIN STORESsession.sqliteDurable entries, tasks, documents;Chord documentsblobs/artifact versions, uploads; named byhash, never changedhistory.sqlitefile-history: rows exist nowhere elsesearch indexderived from the log, rebuildableThe server holds only mirrors (catalog) and instance data (shares, devices, settings).
Figure 1. The session directory of S1. Accent marks the store that makes export and sync hard.
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KindExampleChangesExport and sync
logconversation, documentsappend and updatemust ship, in order
blobartifact v3 contentsnevership the file
derivedfull-text indexrebuiltnever ship, rebuild
plugin storehistory.sqliteupdate in placethe open problem
+
+ +
II

The Easy Cases

Why blobs and derived data need no machinery, and what that teaches about the hard case.

+

The artifacts plugin saves a new version. It writes the content as a blob first, then commits a document that names +the blob's hash. The order is the whole trick.

+
artifact toolblobs/session.sqlitewrite sha256:9f3a… (tmp, fsync, rename)seq 41: chart.versions[2] = { blob: "sha256:9f3a…" }seq 42: entry 812 "chart v3 saved"
Figure 2. Saving artifact "chart" version 3. The blob exists before anything refers to it.
+

An export reads the log at some head, say 42, and copies every file in blobs/. Any blob a record up to 42 refers to +was written before that record, so it is in the directory. The session does not have to stop. An import replays the +log into a fresh session.sqlite, copies the blobs and checks their hashes.

+

Derived data rebuilds itself

+

The artifacts plugin also keeps a full-text index over artifact contents, in its own SQLite file under +derived/artifacts/, because searching documents one by one is too slow. The index holds nothing of its own: every +row can be recomputed from the artifact documents and the blobs they name. So it is never exported or synced. A +routine compares the index with the documents whenever the object starts and fills in whatever is missing:

+
const Artifact = defineState("artifact", {
+  schema: obj({ title: str(), type: str(), versions: arr(obj({ blob: str(), size: int(), at: int() })) }),
+  version: 1, keyed: true, persist: true,                         // key = artifact name
+  initial: () => ({ title: "", type: "text", versions: [] }),
+  checkpointWhen: () => true,
+});
+
+host.serve(ArtifactStore, (_, self) => {                          // local, input-less
+  const artifacts = self.state(Artifact);
+  const blobs = self.get(SessionBlobs, {});                       // app layer: the session's blob store
+  const info = self.get(SessionInfo, {});
+  const index = self.open(async () => openIndex(join(info.init.value.dir, "derived/artifacts/index.sqlite")),
+    (db) => db.close());
+
+  self.routines.loop("catch-up", async (ctx) => {                 // the index repairs itself
+    for await (const name of artifacts.keys()) {
+      const a = await artifacts.get(name);
+      if (a && index.value.versions(name) !== a.versions.length)
+        index.value.put(name, a.versions.length, await blobs.read(a.versions.at(-1)!.blob, ctx));
+    }
+  });
+
+  return {
+    save: async ({ name, title, type, content }, ctx) => {
+      const bytes = utf8(content);
+      const blob = await blobs.put(bytes, ctx);                   // 1. content first
+      await self.tx(ctx, async (tx) => {                          // 2. the document naming it
+        const a = await tx.draft(artifacts, name);
+        a.title = title; a.type = type;
+        a.versions.push({ blob, size: bytes.length, at: Date.now() });
+      });
+      const version = (await artifacts.get(name))!.versions.length;
+      index.value.put(name, version, content);                    // 3. derived, best effort
+      return { version };
+    },
+    search: ({ query }) => index.value.search(query),
+  };
+});
+

The same routine covers three situations without any special code:

+
One routine, three situations

Normal save

save writes the blob, commits the document, then updates the index. The routine finds nothing to do.

+

Crash after the commit

The process dies between step 2 and step 3. The document says version 3, the index still has version 2. On the next +start the routine sees the difference, reads the blob of version 3 and indexes it.

+

Import or sync

On the desktop derived/artifacts/ does not exist. openIndex creates an empty index; the routine finds every +artifact missing and indexes them all from the documents and blobs that came with the session.

+
+
Cost of the catch-up

This routine visits every artifact on each start: fine for hundreds, slow for millions. A large derived store would +keep the last position it applied and read only the changes after it (the outboxes of chapter V give exactly that).

+
+

Both cases work because nothing ever changes in place: a blob is final when written, and a derived index has no data +of its own. history.sqlite breaks both assumptions. Its rows change in place, and they are the only copy.

+
+ +
III

Two Stores, No Common Order

What goes wrong when history.sqlite and the log travel separately.

+

Suppose every store ships its own changes to the desktop as it pleases: session.sqlite in batches of commits, +history.sqlite as a stream of its own. The agent edits x.ts; the edit is recorded in both stores.

+
One edit, two stores, shipped separately

On the laptop

The tool records the edit in history.sqlite (row: path x.ts, entry 812), then Durable commits entry 812 at seq +42. On the laptop both exist and agree.

+

The history stream arrives first

The desktop has received the history.sqlite changes, including the row, but the log only up to seq 41. The row +says entry 812 made the edit; the desktop has no entry 812.

+

Mario takes over now

The laptop's lid is closed; the desktop becomes the driver. Its agent writes the next entry, which gets id 812: a +different message. The row now points to the wrong entry, permanently.

+

The opposite order

If the log arrives first instead, entry 812 says "x.ts edited" and the history has no row for it. Either way the +desktop holds a state the laptop never had.

+
+

The fault is not in either store. Each one is consistent on its own. What is missing is an order across stores: +which change of history.sqlite happened before which commit of the log.

+
+ +
IV

One Counter for the Session

A single sequence of positions that every store takes from when it commits.

+

The session gets one counter, kept in memory by p3's sync layer on the driver. Every store, in the transaction that +commits a change, takes the next position and writes the change with that position into its own outbox. Positions +increase across all stores of the session, so they order everything.

+
edit toolcountersync layer, memoryhistory.sqliteplugin storesession.sqliteBEGIN; INSERT row (x.ts, entry 812)next()7outbox row (7, changeset); COMMITcommit entry 812next()8seq 42 with position 8, same transaction
Figure 3. The same edit with positions. Each store writes its outbox inside its own transaction.
+

After the edit the two outboxes look like this:

+ + + + + + + + + + + + + + + + + + +
StorePositionChange
history.sqlite7changeset: insert row (x.ts, entry 812)
session.sqlite8seq 42: entry 812
+

The sync layer cuts a batch at a position P: every change of every store with a position up to P. A replica +applies a batch in position order, store by store as the positions say. After applying a batch it holds exactly the +state the driver had at P, for all stores together. The bad states of the previous chapter cannot happen: a replica +has position 8 only together with position 7.

+
DRIVERREPLICAhistory outbox7, 11, 12log outbox8, 9, 10, 13cut at 12nothing ≤ 12 still committingapply in orderhistory 7, log 8 9 10, history 11 12batch ≤ 127 8 9 10 11 12uploadPosition 13 waits for the next batch, together with whatever else comes after 12.
Figure 4. Shipping. Batches are cut across stores; the replica applies them in position order.
+

Cutting needs one rule. The counter hands out position 12 to a store whose transaction has not committed yet, +while position 13 already committed elsewhere. A cut at 13 would miss 12. So the counter remembers the positions it +handed out that are not committed yet, and a cut goes only up to just below the smallest of them. A transaction that +fails gives its position back as a gap; gaps are fine, order is what counts.

+
+ +
V

Capturing a SQLite Change

How a plugin writes plain SQL and still produces a change record another database can apply.

+

A plugin store must turn its writes into change records. For JSONL that is the appended lines themselves. For SQLite +it would be a burden on every plugin, except that SQLite has the session extension: it watches tables and turns +whatever SQL ran into a changeset, a compact list of row inserts, updates and deletes. Another database applies +it. node:sqlite exposes it. This ran on slayer with Node 24.4.1:

+
import { DatabaseSync } from "node:sqlite";
+const laptop = new DatabaseSync(":memory:"), desktop = new DatabaseSync(":memory:");
+// both: CREATE TABLE edits(id INTEGER PRIMARY KEY, path TEXT); CREATE TABLE outbox(pos INTEGER PRIMARY KEY, cs BLOB)
+
+const session = laptop.createSession({ table: "edits" });
+laptop.exec("BEGIN");
+laptop.exec("UPDATE edits SET path = 'b/' || substr(path, 3) WHERE path GLOB 'a/*'");  // plain SQL
+const cs = session.changeset();                                                      // 54 bytes for two rows
+laptop.prepare("INSERT INTO outbox VALUES (?, ?)").run(7, cs);                       // same transaction
+laptop.exec("COMMIT");
+
+desktop.applyChangeset(cs);   // desktop now has b/x and b/y
+

So p3 could offer plugins a session SQLite as a store kind. The plugin opens it, writes SQL in transactions, and the +wrapper does the rest: start a session on the plugin's tables, take a position before COMMIT, write the changeset to +the outbox table, commit. The plugin never sees positions or outboxes.

+
Limits of changesets

Rows, not commands. Renaming a/ to b/ over ten thousand rows is ten thousand row changes in the changeset +(tens of bytes each), not one command. Still logical rows, never database pages. +No schema changes. CREATE TABLE and ALTER TABLE are not in changesets. A plugin's schema migrations must be +versioned and run on every replica before changes made with the new schema are applied. +Cloudflare. The Durable Object SQLite API may not expose the session extension; not checked.

+
+
+ +
VI

Crashes

Why the outbox lives inside each store, and how the counter restarts.

+

Why not one shared outbox file for all stores? Because the change and its record must commit together. With a shared +file, the plugin commits its rows, the process dies before the changeset reaches the shared file, and the changeset +is gone: it existed only in memory. The rows are on disk and will never be shipped. Each store therefore keeps its +outbox inside itself, in the same transaction as the change: an outbox table in a SQLite database, the record in +session.sqlite's commit, the lines themselves in a JSONL file.

+
The laptop dies in the middle of a turn

Before the crash

Positions 1 to 14 are handed out. 7, 11, 12 committed in history.sqlite; 8, 9, 10, 13 in session.sqlite; 14 was +still in a history.sqlite transaction. Batch ≤ 12 was uploaded.

+

The crash

Position 14's transaction never committed, so neither its rows nor its outbox record exist. Nothing half-done is +left in any store.

+

Restart

The sync layer reads the highest position in every outbox (12 and 13) and starts the counter at 14. Position 13 is +committed and not yet uploaded; it goes into the next batch.

+

Nothing is lost or reordered

Every committed change has its record; every record has its change. The next batch continues at 13.

+
+
+ +
VII

What Each Store Needs, and What Chord Provides

The store kinds p3 would offer plugins, and the small part that falls to Chord.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Store kindChange recordApplied on a replica bySnapshot (export, compaction)
Durable/Chord storagethe commit's writesreplayexport()
blob directory"added hash" plus the byteswriting the filethe files
append-only JSONLthe appended linesappendingthe file
plugin SQLitechangeset, in the same transactionapplyChangesetVACUUM INTO
files changed in placenonenonecannot sync: derived data only
+

Export and import use the same pieces. An export takes a snapshot of every store at one cut position P (the +log's export(), VACUUM INTO for each plugin database, the blob files, the JSONL files) and writes them into one +archive with P in its manifest. An import restores each snapshot into a fresh session directory. Later changes, on a +replica, continue from P.

+

Chord's part is small. Its storage (and Durable's) must be wrappable: the sync layer sees every commit's writes +in order and gets a hook inside the commit to take a position and write the outbox record in the same transaction. +And the storage needs export(), replay() and head(). The counter, outboxes, batches, store kinds and the +session directory are p3's.

+

One consequence for an open question: ordering no longer needs Chord's documents to live in Durable's storage. If +they had their own file, it would be one more store with its own outbox, ordered by the same counter. Sharing the +storage would only save a file.

+
+ +
∴

Open Questions

What this proposal needs decided before anything is built.

+

The proposal: a session is a directory of stores; p3's sync layer orders every change of every store through one +position counter; each store kind records its changes in its own transaction; plugins get store kinds (SQLite through +changesets, JSONL, blobs) in the session directory; files changed in place may hold only derived data. Chord provides +a wrappable storage with export, replay and head.

+

Needs a decision

+
    +
  • The direction. Is this the model for p3 sessions?
    • +
  • Store kinds. Which kinds p3 offers plugins at first (SQLite and blobs, or JSONL too).
    • +
  • Shared storage. Whether Chord's persisted session documents still share Durable's storage, now that ordering +does not need it.
    • +
+

To verify

+
    +
  • The session extension's cost on large tables and wide updates (a benchmark of the 10k-row rename).
    • +
  • Whether Cloudflare's Durable Object SQLite exposes changesets; if not, what a plugin store means there.
    • +
+
+ +
A

Glossary

+

Batch: all changes of all stores up to one position. Blob: a file named by the hash of its content, never +changed. Changeset: SQLite's record of the rows a transaction inserted, updated or deleted. Cut: choosing the +position a batch ends at; never past a position still committing. Derived data: data rebuilt from other stores, +never shipped. Driver: the instance that writes a session. Outbox: a store's own record of its changes with +positions, written in the same transaction. Position: a number from the session's counter, taken by every change. +Replica: a copy of a session on another instance that applies batches. Store: one place a session keeps data.

+
+ +

One Session, Many Stores · first edition · 2026-10-21 · based on session-data-scenarios.md and the redesign log

+
+ + diff --git a/apps/p2/docs/p3-reviews/session-sync-prototype/demo.ts b/apps/p2/docs/p3-reviews/session-sync-prototype/demo.ts new file mode 100644 index 000000000..c8aeaa2e4 --- /dev/null +++ b/apps/p2/docs/p3-reviews/session-sync-prototype/demo.ts @@ -0,0 +1,96 @@ +// Real-world example for rule 1: the file-history plugin keeps every file edit of session S1 in its own SQLite +// database next to the session's log. Both sync to a desktop replica through one position sequence. +// Run: node demo.ts (Node 24). Stand-in: `log` plays session.sqlite (Durable's entries) with the same wrapper; the +// real one would take its position inside Durable's storage commit. +import { mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { applyBatch, type Batch, cutBatch, SessionSqlite, Sequencer } from "./store.ts"; + +const root = "/tmp/session-sync-demo"; +rmSync(root, { recursive: true, force: true }); + +function openSession(machine: string, driver: boolean) { + const dir = join(root, machine, "sessions", "S1"); + mkdirSync(join(dir, "plugins", "file-history"), { recursive: true }); + const stores = { + log: new SessionSqlite(join(dir, "session.sqlite")), + history: new SessionSqlite(join(dir, "plugins", "file-history", "history.sqlite")), + }; + // Schemas: DDL is not in changesets, so every machine runs the same versioned setup when it opens the store. + stores.log.db.exec("CREATE TABLE IF NOT EXISTS entries(id INTEGER PRIMARY KEY, text TEXT NOT NULL)"); + stores.history.db.exec(`CREATE TABLE IF NOT EXISTS edits( + id INTEGER PRIMARY KEY, path TEXT NOT NULL, entry INTEGER NOT NULL, before TEXT, after TEXT)`); + // The sequencer continues above everything any store has seen (outbox or applied), also after a crash. + const sequencer = new Sequencer(Math.max(...Object.values(stores).map((s) => s.head())) + 1); + if (driver) for (const s of Object.values(stores)) s.attach(sequencer); + return { stores, sequencer }; +} + +// ---- the plugin's code: plain SQL inside store.transaction ---- +const recordEdit = (history: SessionSqlite, e: { path: string; entry: number; before: string; after: string }) => + history.transaction((db) => + db.prepare("INSERT INTO edits (path, entry, before, after) VALUES (?, ?, ?, ?)").run(e.path, e.entry, e.before, e.after), + ); +const renameDir = (history: SessionSqlite, from: string, to: string) => + history.transaction((db) => + db.prepare("UPDATE edits SET path = ? || substr(path, ?) WHERE path GLOB ?").run(to, from.length + 1, `${from}*`), + ); +const commitEntry = (log: SessionSqlite, id: number, text: string) => + log.transaction((db) => db.prepare("INSERT INTO entries (id, text) VALUES (?, ?)").run(id, text)); + +const show = (label: string, s: { stores: Record }) => { + console.log(` ${label} entries:`, s.stores.log.db.prepare("SELECT id, text FROM entries ORDER BY id").all().map((r) => `${r.id} ${r.text}`)); + console.log(` ${label} edits: `, s.stores.history.db.prepare("SELECT path, entry FROM edits ORDER BY id").all().map((r) => `${r.path}@${r.entry}`)); +}; +const outboxes = (s: { stores: Record }) => + Object.entries(s.stores).map(([n, st]) => `${n}: [${st.changesSince(0).map((c) => `${c.pos}(${c.change.length}B)`).join(" ")}]`).join(" "); +const upload = (b: Batch): Batch => JSON.parse(JSON.stringify(b, (_, v) => (v instanceof Uint8Array ? { b64: Buffer.from(v).toString("base64") } : v)), (_, v) => (v && typeof v === "object" && "b64" in v ? new Uint8Array(Buffer.from(v.b64, "base64")) : v)); + +// ---- 1. the agent works on the laptop ---- +console.log("1. laptop: an agent turn writes two files, then renames a/ to b/"); +const laptop = openSession("laptop", true); +commitEntry(laptop.stores.log, 811, "user: move the helpers into a/"); +recordEdit(laptop.stores.history, { path: "a/x.ts", entry: 812, before: "", after: "export const x = 1;" }); +recordEdit(laptop.stores.history, { path: "a/y.ts", entry: 812, before: "", after: "export const y = 2;" }); +commitEntry(laptop.stores.log, 812, "tool result: wrote a/x.ts, a/y.ts"); +renameDir(laptop.stores.history, "a/", "b/"); +commitEntry(laptop.stores.log, 813, "tool result: renamed a/ to b/"); +console.log(" outboxes:", outboxes(laptop)); + +// ---- 2. a transaction that dies before COMMIT leaves nothing behind ---- +console.log("2. laptop: the process dies in the middle of a history transaction"); +try { + laptop.stores.history.transaction((db) => { + db.prepare("INSERT INTO edits (path, entry, before, after) VALUES ('b/z.ts', 814, '', 'z')").run(); + throw new Error("simulated crash before COMMIT"); + }); +} catch (error) { + console.log(" ", (error as Error).message, "-> rows and record both rolled back; outboxes:", outboxes(laptop)); +} + +// ---- 3. upload a batch, apply it on the desktop ---- +console.log("3. sync: cut at the safe position, upload, apply on the desktop in position order"); +const batch = cutBatch(laptop.stores, laptop.sequencer, 0); +console.log(" batch:", batch.records.map((r) => `${r.pos}:${r.store}`).join(" "), `upTo=${batch.upTo}`); +const desktop = openSession("desktop", false); +applyBatch(desktop.stores, upload(batch)); +for (const s of Object.values(laptop.stores)) s.trim(batch.upTo); +show("laptop ", laptop); +show("desktop", desktop); + +// ---- 4. the naive variant: a separate outbox written after COMMIT ---- +console.log("4. contrast: record appended to a separate file after COMMIT, process dies in between"); +laptop.stores.history.db.prepare("INSERT INTO edits (path, entry, before, after) VALUES ('b/w.ts', 813, '', 'w')").run(); +// ...appendFileSync(outbox, changeset) never runs: the row is on disk, its change exists nowhere. +console.log(" laptop edits:", laptop.stores.history.db.prepare("SELECT count(*) AS n FROM edits").get()?.n, "desktop edits:", desktop.stores.history.db.prepare("SELECT count(*) AS n FROM edits").get()?.n, "-> diverged for good, no record to resend"); +laptop.stores.history.db.prepare("DELETE FROM edits WHERE path = 'b/w.ts'").run(); + +// ---- 5. restart and takeover ---- +for (const s of Object.values(laptop.stores)) s.close(); +for (const s of Object.values(desktop.stores)) s.close(); +console.log("5. takeover: the desktop reopens S1 as driver; its sequencer continues above every store's head"); +const driver = openSession("desktop", true); +console.log(" next position:", driver.sequencer.safeCut() + 1); +recordEdit(driver.stores.history, { path: "b/x.ts", entry: 814, before: "export const x = 1;", after: "export const x = 3;" }); +commitEntry(driver.stores.log, 814, "tool result: edited b/x.ts"); +console.log(" outboxes:", outboxes(driver)); diff --git a/apps/p2/docs/p3-reviews/session-sync-prototype/store.ts b/apps/p2/docs/p3-reviews/session-sync-prototype/store.ts new file mode 100644 index 000000000..4c2a30836 --- /dev/null +++ b/apps/p2/docs/p3-reviews/session-sync-prototype/store.ts @@ -0,0 +1,132 @@ +// Prototype for discussion (rule 1): a session store whose change records commit in its own transaction. +// Not product code. Run: node demo.ts (Node 24). +import { DatabaseSync } from "node:sqlite"; + +/** One per session: hands out global positions; knows which are still inside an open transaction. */ +export class Sequencer { + #next: number; + readonly #inFlight = new Set(); + constructor(start: number) { + this.#next = start; + } + take(): number { + const pos = this.#next++; + this.#inFlight.add(pos); + return pos; + } + release(pos: number): void { + this.#inFlight.delete(pos); + } + /** The highest position P such that nothing at or below P is still committing. */ + safeCut(): number { + return this.#inFlight.size > 0 ? Math.min(...this.#inFlight) - 1 : this.#next - 1; + } +} + +export type Change = { pos: number; change: Uint8Array }; + +/** + * A plugin's own SQLite database in the session directory, made syncable. The plugin writes plain SQL inside + * `transaction`; the store records the changed rows (SQLite session extension) in `_outbox`, in the same transaction. + */ +export class SessionSqlite { + readonly db: DatabaseSync; + #sequencer: Sequencer | undefined; + + constructor(path: string) { + this.db = new DatabaseSync(path); + this.db.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS _outbox(pos INTEGER PRIMARY KEY, change BLOB NOT NULL) STRICT; + CREATE TABLE IF NOT EXISTS _applied(id INTEGER PRIMARY KEY CHECK (id = 0), pos INTEGER NOT NULL) STRICT; + `); + } + + /** Driver: the session's sequencer, attached once every store is open (it starts above all their heads). */ + attach(sequencer: Sequencer): void { + this.#sequencer = sequencer; + } + + /** Driver: run `fn` in one transaction; its rows and their change record commit together or not at all. */ + transaction(fn: (db: DatabaseSync) => T): T { + const sequencer = this.#sequencer; + if (!sequencer) throw new Error("not the driver: no sequencer attached"); + const session = this.db.createSession(); // tracks every table with a primary key + let pos: number | undefined; + this.db.exec("BEGIN IMMEDIATE"); + try { + const result = fn(this.db); + const change = session.changeset(); // taken before the outbox insert, so it holds only fn's rows + if (change.length > 0) { + pos = sequencer.take(); + this.db.prepare("INSERT INTO _outbox (pos, change) VALUES (?, ?)").run(pos, change); + } + this.db.exec("COMMIT"); + return result; + } catch (error) { + this.db.exec("ROLLBACK"); // the rows and the record vanish together; the position becomes a gap + throw error; + } finally { + session.close(); + if (pos !== undefined) sequencer.release(pos); + } + } + + /** Driver: change records after `pos`, for upload. */ + changesSince(pos: number): Change[] { + return this.db + .prepare("SELECT pos, change FROM _outbox WHERE pos > ? ORDER BY pos") + .all(pos) + .map((row) => ({ pos: Number(row.pos), change: row.change as Uint8Array })); + } + + /** Driver: drop records the server has. */ + trim(upTo: number): void { + this.db.prepare("DELETE FROM _outbox WHERE pos <= ?").run(upTo); + } + + /** Replica: apply one change; idempotent through `_applied`. */ + apply({ pos, change }: Change): void { + if (pos <= this.applied()) return; + this.db.exec("BEGIN IMMEDIATE"); + try { + this.db.applyChangeset(change); + this.db.prepare("INSERT INTO _applied (id, pos) VALUES (0, ?) ON CONFLICT(id) DO UPDATE SET pos = excluded.pos").run(pos); + this.db.exec("COMMIT"); + } catch (error) { + this.db.exec("ROLLBACK"); + throw error; + } + } + + applied(): number { + return Number(this.db.prepare("SELECT pos FROM _applied WHERE id = 0").get()?.pos ?? 0); + } + + /** Highest position this store has seen (as driver or replica): the sequencer restarts above the maximum of all stores. */ + head(): number { + const outbox = Number(this.db.prepare("SELECT max(pos) AS pos FROM _outbox").get()?.pos ?? 0); + return Math.max(outbox, this.applied()); + } + + close(): void { + this.db.close(); + } +} + +export type Batch = { upTo: number; records: { store: string; pos: number; change: Uint8Array }[] }; + +/** Sync layer, driver side: every store's records after `uploaded`, up to the safe cut, merged by position. */ +export function cutBatch(stores: Record, sequencer: Sequencer, uploaded: number): Batch { + const upTo = sequencer.safeCut(); + const records: Batch["records"] = []; + for (const [store, s] of Object.entries(stores)) + for (const r of s.changesSince(uploaded)) if (r.pos <= upTo) records.push({ store, ...r }); + records.sort((a, b) => a.pos - b.pos); + return { upTo, records }; +} + +/** Sync layer, replica side: apply in global position order, interleaving the stores. */ +export function applyBatch(stores: Record, batch: Batch): void { + for (const r of batch.records) stores[r.store].apply(r); +} diff --git a/apps/p2/docs/p3-reviews/store-bulk-migrations.md b/apps/p2/docs/p3-reviews/store-bulk-migrations.md new file mode 100644 index 000000000..fc182ab63 --- /dev/null +++ b/apps/p2/docs/p3-reviews/store-bulk-migrations.md @@ -0,0 +1,106 @@ +# The store: bulk operations and migrations (S17, proposal) + +Status: **decided (owner, 2026-10-21; decision log "S17")** with these changes: no JSON-field operands and no `json.set`/`json.remove`; per-row column updates are `tx.update(table, key, set)` records; JSON-part changes migrate on access; migration steps are `update` (one `updateWhere`) or `rows` (scan + `tx.update` in chunks). Builds on the decided store (decision log "The store", +S1-S16): real tables, keys and indexed columns typed text or integer, the rest of a row in a JSON column `row`; text in +JavaScript order (SQLite: UTF-16BE BLOBs for key and indexed text); every write goes through one commit step and is +recorded in the outbox; replicas apply records without plugin code; migrations run only where the owning code runs. + +## 1. Problem + +Two kinds of writes touch many rows at once, and recording them as one full-row put per row is expensive: + +``` +rename a/ → b/ over 10,000 edits of ~4 KB → 10,000 puts, ~40 MB in the outbox (S17, Astra) +v2 of Edits adds `size` (default 0) → every row rewritten: same cost +retire a document's 300 revisions → 300 delete records +``` + +And a migration of a large table inside one commit would hold the commit step (every owner's commits) for its whole +duration. + +## 2. Bulk operations: recorded as operations, executed by the store on every replica + +```ts +await self.tx(async (tx) => { + await tx.updateWhere(Edits, (e) => prefix(e.path, "a/"), { path: replacePrefix("a/", "b/") }); + await tx.deleteWhere(Revisions, (r) => eq(r.document, docId)); +}, ctx); +``` + +Outbox: + +``` +{ seq: 57, ops: [ + { op: "update", table: "file-history/edits", where: { prefix: ["path", "a/"] }, + set: { path: { replacePrefix: ["a/", "b/"] } }, count: 10000 }, + { op: "delete", table: "pi-durable/revisions", where: { eq: ["document", 41] }, count: 300 } ] } +``` + +- `where` is the query DSL's predicate (S12 scope); `set` takes values or expressions from a fixed set: + `value`, `add(n)`, `max(v)`, `min(v)`, `replacePrefix(from, to)`, `concat(…)`, `coalesce(v)`, and for the JSON part + `json.set(path, expr)`, `json.remove(path)`. +- Deterministic by construction: only the store's own expressions, applied to the same prefix of the log. A replica + executes the operation itself (compiled to one SQL `UPDATE`/`DELETE` on SQLite, a loop on the memory backend), no plugin + code. +- `count` (rows affected on the driver) is recorded; a replica that affects a different number stops applying that + store and restores it from a snapshot (divergence is loud, not silent). One integer per operation. +- Cost: the 10k-row rename is ~150 bytes in the outbox instead of ~40 MB; one SQL statement on each side. +- Not expressible (arbitrary JS per row): the plugin reads, computes and `put`s rows (full rows, as today). + +## 3. Table changes (recorded, rendered by the store) + +``` +{ op: "table", table: "file-history/edits", version: 1, key: ["id"], + columns: { id: "text", path: "text", entry: "int", at: "int" } } // create; non-indexed fields live in `row` +{ op: "alter", table: "file-history/edits", version: 2, + add: { size: { type: "int", default: 0 } } } // ALTER TABLE ADD COLUMN ... DEFAULT 0 +{ op: "alter", …, rename: { at: "time" } } { op: "alter", …, drop: ["entry"] } +{ op: "drop", table: "file-history/edits" } +``` + +- Only key and indexed columns are real columns; adding, removing or renaming a field in the JSON part changes no table. +- Changing a key or a column's type is a table rebuild: the store records `{ op: "rebuild", … }` and does it as + create-copy-drop-rename on every replica (no rows in the outbox). +- Indexes are not recorded: each host creates the indexes its code declares (S-decided: local). + +## 4. Migrations + +```ts +const Edits = defineTable("edits", { + version: 3, + schema: obj({ id: str(), path: str(), entry: int(), time: int(), size: int(), meta: json(MetaV3) }), + key: ["id"], + indexes: { byPath: ["path", "time"] }, + migrations: { + 2: { alter: { add: { size: { type: "int", default: 0 } } } }, // table change only + 3: { alter: { rename: { at: "time" } }, + update: { set: { meta: json.set("$.source", value("agent")) } } }, // bulk op, no row function + // 4: { rows: (row) => ({ ...row, meta: rebuild(row.meta) }) } // arbitrary JS: full-row puts + }, +}); +``` + +- A migration is a list of steps per version: table changes, bulk operations, and only when nothing else fits a row + function (`rows`), which becomes full-row puts. +- Runs eagerly when the owning object activates, on the driver, before the object goes live (a readiness gate: calls + wait, as for any activating object). Old code finding a newer table version refuses to open it, data intact (as R15). +- Large row-function migrations run in chunks of N rows per commit, so the commit step stays short for other owners; + the table's version record moves `migrating → ready` around them. A replica without the plugin just applies the + chunks; a replica with the plugin's code treats the table as not ready until `ready`. +- Chord documents keep their own per-document `migrate` on access; Durable versions its records itself. Table + migrations are for tables only. + +## 5. What it costs and adds + +- The store gets an expression evaluator twice (SQL compiler, memory backend), checked by one conformance suite; + about 300 lines (estimate). +- The DSL grows by `updateWhere`/`deleteWhere` and the `set` expressions; no change to queries. +- SQLite specifics stay inside the backend: `replacePrefix` on a UTF-16BE BLOB column is a byte prefix of + `2 × length(from)`; `json.set` uses SQLite's JSON functions on the `row` column; the memory backend does the same in JS. + +## 6. Open questions (one at a time) + +1. Bulk operations recorded as operations executed on every replica, with a row count check (above), or resolved to + per-row records on the driver (simple replicas, large outbox)? +2. The expression set for `set`. +3. Migrations: eager with a readiness gate and chunking (above), or lazy per row for changes to the JSON part only. diff --git a/apps/p2/docs/p3-reviews/store-proposal.md b/apps/p2/docs/p3-reviews/store-proposal.md new file mode 100644 index 000000000..682a84b47 --- /dev/null +++ b/apps/p2/docs/p3-reviews/store-proposal.md @@ -0,0 +1,183 @@ +# The store: one transactional table store under Durable, Chord and plugins (proposal under review) + +> **Superseded (2026-10-21):** the decided design is the book `packages/chord/docs/store/book.md` and the decision +> log entries "The store", "S1" to "S17", "Blobs and derived data", "Index definitions are recorded". This page is the +> state that went into the review. + +Status: direction accepted by the owner on 2026-10-21 (decision log, "The store"); details below are open for review. +Working name "the store" (the owner rejected "session table store": it is a general embedded store; a session is one +use). Background and measurements: `session-data-scenarios.md`, prototype `session-sync-prototype/`, explainer +`session-sync-explainer/` (option E, now superseded by this). + +## 1. Problem + +**P1 (owner):** a session is self-contained: everything of it lives in its storage plus its directory; it can be +exported from the session alone and imported into another p3 instance; it can be synced to Radius and continued on +another device (Armin's "Session Sync, second edition": a decorator captures every commit into an outbox, batches go +to Radius, a replica replays them, one driver per session, two writers fork the session). + +That works for one storage (Durable's `session.sqlite`). It does not extend to: + +- **Chord's persisted documents** (today a second storage with its own sequence); +- **plugins' own storage** in the session directory (a SQLite file for performance, a JSONL log): no outbox, and no + order relative to the session's commits. + +The failure, traced (file-history plugin, own `history.sqlite`): + +``` +laptop t1 history.sqlite: row { path: "x.ts", entry: 812 } + t2 session.sqlite seq 42: entry 812 +replica received history up to t1, the log only up to seq 41 → the row points to a missing entry +takeover now: the desktop writes another entry 812 → the row points to the wrong entry, permanently +``` + +## 2. Discarded solutions (in the order discussed) + +| # | Idea | Why discarded | +| --- | --- | --- | +| A | Plugins' own databases may hold only derived data (rebuilt, never synced); source data in documents, entries, blobs | owner: too limiting; some plugins need their own query power and write model | +| B | Export plugin directories as opaque files | needs a quiescent session; cannot sync incrementally (page shipping) | +| C | Plugin export/import hooks | for sync each plugin would build its own incremental log | +| D | Plugin record logs in the session log, the plugin's database as a projection (`apply`) | plugin must write deterministic `apply` and keep every record version forever | +| E | Many stores, each with an in-transaction outbox, one global position counter, a safe cut | works (prototype), but many outboxes and a cut rule; SQLite changesets are per row (10k-row rename 259-418 KB), DDL not captured, Cloudflare support unknown | +| F | Generalized session SQLite with changesets, or with recorded SQL statements verified by a patchset hash | SQLite-only; statements risk nondeterminism (`random()`, unordered `INSERT ... SELECT`, SQLite versions) | +| G | Raw SQL reads, controlled writes | SQLite-only backends; reads need a write guard | +| H | An IndexedDB-shaped API only | owner: too limited for many queries (aggregates, joins, multi-range filters) | +| I | Chord documents inside Durable's `Storage` (shared storage, earlier reviewer proposal) | covers Chord, not plugins; Durable's write grammar only | + +## 3. The design + +### Layers + +``` +backend SQLite (Node, Durable Objects, sqlite-wasm) | memory | IndexedDB + rows, declared indexes, one atomic write batch +store defineTable, query DSL, controlled writes, commit queue, seq, outbox, export/replay + ├─ Durable adapter StoreStorage implements Durable's Storage on tables "pi-durable/*" + ├─ Chord documents on tables "chord/*" + └─ plugin tables self.table(...) in their owning objects +``` + +Durable's existing storages (memory, SQLite, JSONL, IndexedDB) stay unchanged; `StoreStorage` is added next to them +and must pass Durable's storage conformance suite. One store per session (on the session's worker); the server may +have its own store for instance data. + +### Documents engine (S13, decided) + +One engine in the store for persisted documents, used by Durable's `StoreStorage` and directly by Chord: address +`(namespace, owner, kind, key?)` with an opaque owner; `find`/`load`/`list` outside the step, `create`/`change`/ +`retire` inside it; API and rules in the decision log ("S13"). + +### Tables + +```ts +const Edits = defineTable("edits", { + version: 2, + schema: obj({ id: str(), path: str(), entry: int(), at: int(), size: int() }), + key: ["id"], + indexes: { byPath: ["path", "at"], byEntry: ["entry"] }, + migrate: (row, from) => ({ ...row, size: row.size ?? 0 }), +}); + +host.serve(FileHistoryStore, (_, self) => { // input-less local object (P3) + const edits = self.table(Edits); // address: (host, object type, "edits") + … +}); +``` + +- **`json(schema)` columns (owner, 2026-10-21, proposal):** a column that holds a JSON value typed by its schema but is + opaque to the store: not indexable, not usable in `where`/`orderBy`/`groupBy` (a type error), read back typed (no + manual `parseJson` casts as in Durable's SQLite storage today), not validated on write (R9), part of the table's + fingerprint. Scalar columns are what queries and indexes use. Example: `pi-durable/entries` = + `obj({ id: int(), conversationId: int(), headOf: opt(int()), commitSeq: int(), record: json(EntryRecord) })`. + Under the real-tables layout this is exactly Durable's current SQLite shape (index columns plus `record TEXT`). +- A table belongs to one owning object, like a document: only that object reads and writes it; address (host, + object type, table name), so an input change cannot orphan data; other plugins see the data only through the + object's members. +- Rows are JSON objects. A row missing an indexed field is not in that index (on every backend: the partial-index + rule; Durable's head-entry index uses it). +- Indexes are local: created by the owning code where it runs; never recorded. +- Migrations run only where the owning code runs (the driver); rewritten rows are ordinary puts in the outbox; old code + reading a row of a newer version is refused, data intact (as R15 for documents). + +### Queries (small DSL) and controlled writes + +```ts +q.from(Edits).where((e) => eq(e.path, "src/app.ts")).orderBy((e) => e.at, "desc").limit(20).after(cursor) +q.from(Edits).groupBy((e) => e.path).select((e) => ({ path: e.path, n: count() })) + +await edits.transaction(async (tx) => { // under the owning object's line + const g = await tx.get(Guests, id); + await tx.put(Guests, { ...g, revision: g.revision + 1 }); // read, compute, put: no expressions + await tx.delete(Links, [linkId]); + await tx.deleteWhere(Revisions, (r) => eq(r.document, docId)); // resolved to keys, one delete record per row +}); +``` + +Proposed DSL scope (not yet confirmed by the owner): `get`; `where` with `eq ne lt lte gt gte inArray isNull prefix +and or not`; `orderBy` (several fields), `limit`, cursor; `select` of fields; `groupBy` with `count sum min max`; one +equality `join`. Out: subqueries, `LIKE`/regex/full text, offset paging, computed selects, window functions, `HAVING`, +more than one join. Full-text search stays derived (a local SQLite FTS, never synced). + +### Lines, the commit queue, seq + +| | serializes | duration | count | +| --- | --- | --- | --- | +| object line | a whole transaction callback (read, decide, write) | as long as the callback | per Chord object; per Durable Session | +| commit queue | only the final write batch: adapter checks, writes, seq, outbox record | short, no waits outside the store | one per store | + +```ts +store.commit(async (batch, seq) => { /* reads only through batch; checks; puts and deletes */ }); +``` + +- **Adoption inside the step (S1, decided):** `store.commit(prepare, adopt)`; each owner's synchronous in-memory + adoption runs after the backend commit and before the next step; Durable's `Storage.commit(writes, adopt, + context)` carries it. Listeners run afterwards in each owner's continuation. +- **Completion (S3, decided):** the step ends on the backend's terminal success only (SQLite `COMMIT`, IndexedDB + `complete`); memory stages and swaps; an ambiguous outcome stops the store; after a restart the driver pulls back + its own uploaded batches when its local head is behind. +- The seq is assigned inside the commit step, and the outbox record is written in the same backend transaction as + the rows, so a crash keeps both or neither. +- Durable's seq is the store's seq: `StoreStorage.commit(writes)` runs its checks and stamps `commitSeq`, `createdAt`, + `retiredAt` and revision seqs inside the step, as `SqliteStorage.commit` does inside its SQLite transaction today. + Durable sees gaps (other owners' commits); it only compares seqs (`created_at <= at`, `seq > base`; checked: + no `seq + 1` neighbour arithmetic in the storages or the session). +- Chord documents get the same seq, so they share one timeline with the transcript (outbox, export, "as of" reads). + +### The outbox, sync, export, import + +``` +{ seq: 41, ops: [ + { table: "pi-durable/entries", key: [812], v: 1, row: { … } }, + { table: "file-history/edits", key: ["e1"], v: 2, row: { … } }, + { table: "file-history/edits", key: ["e0"], delete: true } ] } +``` + +- A replica applies ops without any plugin code: it stores rows under (table, key). Tables of plugins that are not + installed travel along untouched. +- Sync: Armin's transport and fencing (batches of outbox records, one driver per session, epochs, fork on conflict). +- Export (S4, decided): pauses commits while it reads, so the archive is the state at one head; import: restore into a fresh store; then the server indexes the + session (catalog mirror). +- Derived data (FTS, caches) is rebuilt where needed, never shipped. + +## 4. Open points + +1. Physical layout: generic rows `rows(tbl, key, v, row)` (no DDL on replicas, JSON parsing on scans) or one real table + per logical table (definitions and changes recorded, DDL and rebuilds on replicas). Measured for generic rows on + 100k rows: indexed lookup 0.07 ms, join + group 129 ms, group by 49 ms. +2. DSL scope (section 3). +3. Migration timing: eager at activation (consistent indexes, one burst) or lazy per row. +4. Durable's `next_id`: a row written per commit (replicates for free) or derived at open from the highest id. +5. Backend details: string order (UTF-16 code units as in JS and IndexedDB vs UTF-8 bytes in SQLite), values that + cannot be index keys in IndexedDB (booleans, null), IndexedDB version upgrades for new indexes. +6. Atomic writes across owners: possible in one batch, but owners commit separately; core invariants stay in one + owner's transaction. + +## 5. Owner's worry: "seq 40 after seq 50" + +Could anything observe, store, upload or apply a commit with a lower seq after one with a higher seq, or act on state +at seq 50 that lacks seq 40? Places to check: seq assignment vs completion of the write; in-memory adoption and +observer notification order across owners (Durable's Session adopts after its commit, Chord objects adopt after +theirs); reads outside transactions between two commits; derived data and routines reacting to commits of different +owners; the outbox cut and upload; export snapshots; replica apply; takeover; async backends (IndexedDB) and the +queue; aborted commits (gaps). diff --git a/apps/p2/docs/p3-reviews/store-review-summary.md b/apps/p2/docs/p3-reviews/store-review-summary.md new file mode 100644 index 000000000..ef49f3666 --- /dev/null +++ b/apps/p2/docs/p3-reviews/store-review-summary.md @@ -0,0 +1,51 @@ +# Store proposal review (Fable, Astra), 2026-10-21: summary for the owner + +Source: `store-proposal.md` at `931df40ea`. Both: direction right; disk, outbox and export order are the seq order by +construction (one serial commit step, rows plus outbox record in one backend transaction). The remaining "seq 40 after +seq 50" cases are outside the backend. Astra's invariant: **each published head denotes one complete committed prefix, +not merely the largest seq somebody has seen.** Nothing below is decided; one at a time. + +| # | Sev. | Finding | Proposed fix | Source | +| --- | --- | --- | --- | --- | +| S1 (decided 2026-10-21: `Storage.commit(writes, adopt, context)`, log "S1") | high | In-memory adoption after the step: A commits 40, B commits 41, B's continuation adopts first; a routine on 41 reads A at 38/39 (Durable adopts after `await storage.commit`, `session.ts:750-765`) | `store.commit(prepare, adopt)`: synchronous adoption inside the step before the next starts; listeners after; needs a small Durable Session integration (the `Storage.commit` signature alone cannot give it) | both | +| S2 (decided 2026-10-21: covered by the existing transaction rule, one sentence in the guide) | high | Forward references across owners: a row at 40 references entry 812 created at 50; a valid prefix (40) dangles, and the id may be reused | reference only committed data (the tool-call entry), or write both in one owner transaction, or an explicit unresolved token | Astra | +| S3 (decided 2026-10-21: terminal success, staged memory, ambiguous stops, pull-back) | critical | Backend request success is not commit completion (IndexedDB puts succeed, the transaction aborts later; IndexedDB `complete` is not durable) | finish the step only on the backend transaction's terminal success; memory backend stages and swaps; ambiguous outcome stops the queue; after restart pull back own uploaded batches | both | +| S4 (decided 2026-10-21: export pauses commits; multi-query reads in one read transaction; no global snapshots) | high | Export or multi-query reads across commits give a state that never existed | export takes one read snapshot inside the queue (or pauses writers at first); document materialization in one read transaction | both | +| S5 (decided 2026-10-21: atomic batch apply; parallel uploads, resubmit until acknowledged, trim after ack; apply only when from = applied head, later batches wait; sync layer) | critical | Replica apply: per-record apply splits a commit; out-of-order batches advance `applied` past a gap; acks advance by max | apply one batch atomically with the applied frontier; validate batch coverage; contiguous ack frontier | both | +| S6 (decided 2026-10-21: covered by S5) | high | Compaction keeps only last ops per (table, key): intermediate states inside a compacted range are not real | a compacted range is one transition: nothing published or taken over mid-range | Astra | +| S7 (decided 2026-10-21: no rollback; the local copy becomes a separate session with a new id) | major | Takeover: a commit already queued lands after the epoch changed | epoch checked inside the step; `not_driver` refuses (not `storage_failed`); the worker goes read-only and restarts from the server copy | Fable | +| S8 (decided 2026-10-21: app concern; §14 non-overlapping handlers; guide patterns) | medium | Slow reaction to 40 overwrites the result of the reaction to 50 | app pattern: coalesce or reject stale results by source revision in the destination owner's transaction | Astra | +| S9 (decided 2026-10-21: persist head, drop Sequencer) | high | Prototype: head derived from the outbox resets after trimming; position counter and safe cut are unnecessary with a serial step | persist the head with the commit; delete `Sequencer` | both | +| S10 (decided 2026-10-21: narrow the contract, derive the counter at open, no reservations) | high | `mintId`: a minted id must not be reused after reopen; neither "max stored id" nor "a row on the next commit" gives that (existing SQLite storage has the same contract gap) | persisted id reservations (blocks) before handing out ids | Astra | +| S11 (decided 2026-10-21: StoreStorage resolves copies before the step, Durable unchanged) | decision | `document.copy` (fork copies a document): Fable: resolve on Durable's Session line before the step (the step must not replay revisions while every owner waits); Astra: resolve inside the transaction as today (no extra layer) | owner chooses | both | +| S12 (decided 2026-10-21: DSL without join, groupBy, select at first) | cut | Start the DSL without `join`, `groupBy`, `select` (aggregate in JS over indexed scans); add on demand | | Fable | +| S13 (decided 2026-10-21: one documents engine in the store, opaque owner; API in the log) | cut | One document engine: Chord's persisted documents as Durable-shaped documents through the same adapter (deletes Chord's own storage layer, tags, `retireByTag`, ~1,000 lines) | | Fable | +| S14 (decided 2026-10-21: one transaction API, self.tx) | cut | One transaction API for tables and documents (not `table.transaction` next to `self.tx`) | | Astra | +| S15 (decided 2026-10-21: kept) | decision | Durable's existing storages: keep (owner) vs `StoreStorage` as the only production storage later (Fable) | | Fable | +| S16 (decided 2026-10-21: real tables; JS/UTF-16 order, SQLite indexed text as UTF-16BE BLOBs; Astra's design) | open points | Generic rows (Astra: choose them); eager migration with a readiness gate, primary keys preserved (both); one key codec for string order and IndexedDB key limits (Fable) | | both | +| S17 (decided 2026-10-21: bulk operations and migrations, log "S17") | cost | Full-row puts: renaming 10k 4 KB rows logs ~40 MB before compression (not 259 KB); revision pruning emits one delete per row; the "short step" is not short for bulk writes, migrations or copies | document; bulk operations later if needed | Astra | + +## S16 notes (2026-10-21, in progress) + +- Owner: real tables (definitions and changes in the outbox); store-maintained index entries rejected (cost). +- String order options: (1) backend-native, (2) UTF-16 everywhere (extra order column in SQLite), (3) code point = + UTF-8 byte order everywhere (owner leans to it; sent to both reviewers). Measured (Node 24.4.1, 200k path-like keys, + 70% ASCII, CJK, fullwidth, emoji): code point comparator equals UTF-8 byte order (verified); sort 113.5 ms (JS + default) vs 139.9 ms (code point comparator, no allocation) vs 182.5 ms (pre-encoded `Buffer.compare`); 100k binary + searches 105.6 vs 123.9 ms; encoding 200k keys to UTF-8 120.9 ms (IndexedDB binary keys). SQLite: no cost (native + order). + +## S16 cost: indexed text as UTF-16BE BLOB vs UTF-8 TEXT (Node 24.4.1, 100k rows, index (path, at, id)) + +| operation | TEXT | BLOB | | +| --- | --- | --- | --- | +| insert 100k rows, one transaction (incl. encoding) | 185.7 ms | 259.5 ms | +40% | +| 20k lookups `path = ?`, newest 20 (incl. decoding results) | 54.7 ms | 71.3 ms | +30% | +| 2k prefix ranges of 100 rows (incl. decoding 200k values) | 78.5 ms | 185.2 ms | +136% | +| page through all, 200 pages of 500 (incl. decoding) | 38.5 ms | 75.1 ms | +95% | +| 2k prefix ranges, rows read from the JSON column (no BLOB decode) | 135.9 ms | 140.7 ms | +4% | +| file size | 9.6 MB | 15.6 MB | +62% | + +SQLite itself is barely slower on BLOBs (+4%); the overhead is the JS encode/decode (`Buffer.from(s, "utf16le").swap16()`) +and twice the bytes for ASCII. Reads that take logical values from the row's JSON instead of decoding the indexed +columns avoid the decode cost; writes and query bounds still encode. diff --git a/apps/p2/docs/p3-reviews/v2-cost-model.md b/apps/p2/docs/p3-reviews/v2-cost-model.md new file mode 100644 index 000000000..3f8bc68d4 --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-cost-model.md @@ -0,0 +1,167 @@ +# Chord v2: the cost model of the state path (before / after) + +Sources: Fable's and Astra's measurements on committed HEAD `c990f587f` (500 KB chat-like value, one small change, +Node 24 on slayer), Durable's store and view (`packages/durable/src/session/transaction.ts`, `harness/view.ts`) and +Chord Delta (`packages/chord/src/delta/README.md`). "After" is the target; nothing here is built yet. + +**Goal:** as little compute and memory as possible while staying correct. Delta was built for exactly this: +immutable revisions with structural sharing, exact ops, appliers that share containers. Its own primitives are +practically free (`diffRevisions` on shared subtrees 0.018 ms, `applyImmutable` per small change 0.004 ms). Almost +all measured cost comes from layers the v2 kernel added around it. + +**Headline now:** one small change to a 500 KB value costs 35 ms with no viewer, 13 ms more per viewer (10 viewers: +130 ms); with SQLite 89 ms with no viewer. **Target:** O(size of the change) per commit and per viewer, independent of +the value's size; 500 KB is touched only when a new viewer takes its first snapshot. + +--- + +## 1. Commit + +**Before** (`documents/owner.ts:614-622`): +```ts +// per commit, per touched document +assertDocumentValue(value); // isJsonValue walk of the whole document: ~2 ms +assertValid("document", name, schema, value); // schema check of the whole document: 6.7-8.5 ms +if (dev) freezeValue(value); // whole-document freeze: 1.7-2.6 ms +``` +**After** (as Durable): Delta's `prepare()` already copied and checked the placed values (only the new data); the +commit adopts by pointer swap. No walk, no schema check, no freeze. The schema is checked by fingerprint (D9), once +per access, O(1). + +| | Before | After | +| --- | --- | --- | +| one appended note to a 500 KB document | ~10-14 ms | O(new note), microseconds | + +## 2. Reads in a transaction + +**Before:** every access validates the whole document (`documents/owner.ts:160`, 6.7-8.5 ms, also for a read-only +transaction: 14.9 ms); `tx.get` after `tx.draft` copies the whole draft (`owner.ts:554`, 6.5 ms). + +**After** (as Durable): only `tx.draft`; reading is drafting (Delta's overlay, no copy); an unchanged draft writes +nothing, also for an absent document (it is not created). Fingerprint check, O(1). + +```ts +// before +const n = await tx.get(notes, id); // copy if a draft is open, plus validation +// after +const n = await tx.draft(notes, id); // the overlay; read it, change it or not; unchanged = no write +``` + +## 3. Storage + +**Before** (`storage/sqlite/storage.ts:61-200`): every commit loads the base, replays every delta since it, +validates after each replay, builds a temporary `MemoryDocumentStorage` (copy, validate, freeze), and copies its +snapshot. Same tiny commit with 0 / 10 / 50 earlier deltas: **17 / 33 / 101 ms** (grows with history). The memory +storage copies every write and validates and freezes the whole resulting document (~3.5-5 ms). JSONL inherits that. + +**After** (as Durable): the storage checks metadata only (incarnation, version, sequence, address) and writes the +delta row (or a base when the document's `checkpointWhen` says so). Replay only when loading, once. The memory +storage keeps values by reference (immutable by convention) and stores ops; no copy, no freeze. + +| | Before | After | +| --- | --- | --- | +| tiny commit, SQLite, 50 earlier deltas | 101 ms | one INSERT of a small JSON row | +| tiny commit, memory | 3.5-5 ms | O(change) | + +## 4. Publishing to subscribers on the same host + +**Before:** the provider validates every exposed member on every change, per binding (`object.ts:360,570,576`, +6.7-8.5 ms each); per-principal views multiply that. + +**After** (as `replicatedState`, which Durable uses): subscribers get the same value and ops by reference. No +validation, no copy. + +## 5. Derives + +**Before:** every recompute walks its result (`derive.ts:115`, `isValue`, 1.7-2.4 ms), then `diffValues` walks both +values (`isJsonValue` twice, 3.5 ms) before `diffRevisions`; with any byte leaf it falls back to `deepEqual` via +`canonicalJson`, recursively (10.6 ms instead of 3.2). + +**After:** no walks; `diffRevisions` directly (structural, identity short-circuit; bytes are atomic leaves; equality +never by stringify). Rule for authors: derives reuse structure (memoize per element), so a recompute is pointer work: + +```ts +// before: 2,000 new objects per commit, nothing shared, a full diff +self.derive("public", [notes], ([n]) => ({ list: n.list.map(({ by, ...rest }) => rest) })); + +// after: 2,000 identical references plus one new one; diffRevisions emits exactly one insert +const redacted = new WeakMap(); +self.derive("public", [notes], ([n]) => ({ list: n.list.map((x) => redacted.get(x) ?? memo(redacted, x, strip(x))) })); + +// better still: model the data so no derive is needed (a document without the private field) +``` + +Large combined views (several documents plus entries, as Durable's `ConversationView`) are maintained by re-pathing the +sources' ops (`harness/view.ts:220-244`), never by recomputing; ops are never interpreted by application code. + +## 6. Encoding and sending + +**Before:** a state frame is planned and stringified twice per send (size check, then send) and four times when +queued; memory links stringify every frame to size it; snapshots are deep-cloned per viewer (`connection.ts:618,666, +894`, 1.2 ms each). + +**After:** encode once per frame (Delta's `encoder()` per stream), take the size from the encoded bytes, keep it with +the queued frame. No clone of snapshots (immutable by convention; the transport serializes anyway). + +| | Before | After | +| --- | --- | --- | +| 500 KB snapshot to one viewer | ~2 x 2 ms encode + 1.2 ms clone | ~2 ms encode, once | +| one small change to one viewer | dominated by validation (~13 ms) | microseconds | + +## 7. Relays + +**Before:** the server decodes and walks relayed frames like its own. + +**After:** the relay reads the frame header (hold id, kind, route) and forwards the body opaquely; no decode, no walk, +no re-encode. + +## 8. Receiving + +**Before:** every received frame is walked fully (`isFrame` / `isValue`): five times on WebSocket (ping check, pong +check, `bye` check, decode, `#receive`), twice on IPC/memory; also for trusted server-worker frames. A 1 MiB byte leaf +costs 85 ms per check (`Reflect.ownKeys` enumerates every index). + +**After:** `JSON.parse` already yields strict JSON, so no JSON-ness walk; frames are checked once by their header; +bytes are recognized in O(1) (type and prototype). Method arguments (and hold inputs) whose caller is not a host +principal are validated once, at the provider, with the provider's schema, also when relayed over trusted IPC; +schemas are compiled once and cached (TypeBox interpreted 5.6 ms vs. compiled 0.012 ms per check). + +## 9. Applying at the consumer + +**Before:** after applying, the consumer validates the whole value per change (`remote.ts:228,282`). + +**After:** `decoder()` then `applyImmutable(replica, ops)`: a new revision sharing structure with the old; listeners +get it. No validation, no clone. 0.004 ms per small change. + +## 10. Calls + +**Before:** arguments: consumer strip, provider strip, validation, clone; results: validation at the provider and the +consumer, clone (full-value echo: 15.4 ms same host, 36.7 ms over memory links); the validation schema is rebuilt on +every check; every hold sends the full contract description, which is parsed and hashed (51 KB contract: 3.6 ms plus +frame walks). + +**After:** arguments validated once at the provider when the caller is not a host principal (compiled schema); results +not validated in production; same-host calls pass values by reference (immutable by convention, as `local`), to be +decided; a hold sends the contract fingerprint and the provider fetches the description once per (link, fingerprint). + +## 11. Memory + +**Before:** per-principal views each hold their own derived copies; frames and snapshots are cloned per viewer; the +memory storage holds copied and frozen rows. + +**After:** one authoritative value per document per host (the tracker); consumer replicas share structure with each +other; derived values share structure with their sources; views of the same variant use one value from the core; +extra copies only for adapted bindings (one cached value per binding). + +--- + +## Decisions this needs + +1. The model as a whole: Delta/`replicatedState` end to end (shared immutable values, ops passed through, opaque + relays, one validation at untrusted ingress). +2. The store as a port of Durable's store design. +3. Only `tx.draft` (no `tx.get`); unchanged drafts write nothing, also for absent documents. +4. Derives reuse structure (documented rule); large combined views re-path ops; no op-rewriting `project`. +5. Same-host remote calls without a copy (immutable by convention), or keep the copy. +6. Argument validation at the provider by the caller's trust, with compiled, cached schemas. +7. Contract descriptions once per (link, fingerprint). diff --git a/apps/p2/docs/p3-reviews/v2-review-astra.md b/apps/p2/docs/p3-reviews/v2-review-astra.md new file mode 100644 index 000000000..13c47caff --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review-astra.md @@ -0,0 +1,183 @@ +# Chord v2 review — Astra + +Reviewed committed `private` HEAD `e5864dd826b145fc78d2165ed78a580dfa0fe9d5`. Concurrent native/retry implementation changes were excluded: inspection and execution used a `git archive` snapshot under `/tmp/chord-astra-head`. The decision log takes precedence over the design. + +**Verdict: not ready as p3's runtime.** There are reproducible authority, failure-propagation, cancellation and compatibility defects in the committed kernel. The remaining registry/reload/job design also needs a small adversarial prototype before porting real plugins. This is not a recommendation to discard the design or to restore Chord v1's distributed reload transaction. + +## Evidence and scope + +- Committed Chord next suite: **412 tests passed in 53 files**. Command: `node /home/badlogic/pi-private/node_modules/vitest/dist/cli.js --run test`, from the snapshot's `packages/chord`, using Node 24.4.1. +- Committed playground: **19 tests passed**, including its shared SQLite cases and five playground cases. Command: `node --conditions=source --test test/playground.test.ts test/document-storage.test.ts`. A playground-local package symlink resolved Chord to the snapshot, not the concurrently edited workspace. +- Separate bounded probes reproduced findings 1–10 below. Probe files are `/tmp/chord-astra-head/astra-probe.ts`, `astra-probe2.ts`, `astra-bindings.ts`, and `astra-performance.ts`; their essential setups/results are recorded below so the findings do not depend on those temporary files surviving. +- Read the v2 design, decision log, walkthrough, implementation notes, kernel/storage/transport code and relevant tests; compared the old Chord specification and guide, p3 functionality map, p2 runtime and representative chat/workspace/sharing/terminals/subagents/codemode integrations, plus Durable's harness, registry and phase-snapshot integration. This is not a full audit of Durable or every p2 plugin. +- No real sessions, production endpoints or credentials were used. No implementation files were edited. Natives, routines, jobs, registries and reload are not implemented in the reviewed snapshot; findings about those explicitly identify design gaps, not failures of the concurrent implementation. + +## Findings + +### 1. Blocker — remote methods receive host authority, not the bound caller + +**Location:** `packages/chord/src/object.ts:535–547`; `connection.ts:1001–1031`; design §5, `ctx.principal` row. + +`LocalBinding.call()` constructs a remote method's `RemoteCallScope` with `this.host.principal`. The authenticated/bound object's principal is not used. Incoming dispatch also starts an ordinary host root, so simply copying the ingress scope's principal would not fix every path. + +**Reproduction:** server principal `{kind:"server"}`, accepted client principal `{kind:"guest"}`, remote factory records its principal, method returns `ctx.principal`. Result: `factory = guest`, **`method = server`**. This also affects `ctx.get`: it acquires subsequent views under host authority. A method that permits an operation when `ctx.principal.kind === "server"` grants it to a guest. Existing playground authorization captures the factory principal, so its passing guest tests do not exercise this boundary. + +**Fix:** use the remote object's authenticated, bound principal for remote method contexts and their `ctx.get`; retain host authority only for principal-free local calls and declared internal references. Test guest → remote method → `ctx.get` on the same host, over a direct link and through a relay, including owner-only denial and attribution. This is a required security correction, not an optional API refinement. + +### 2. Blocker — consumer-supplied schemas can synchronously stall the host before admission + +**Location:** `packages/chord/src/host.ts:274–306,342–348`; `contract.ts:73–75,232–261`; `values.ts:197–209,546–686`. + +An incoming hold reconstructs an executable validation description from peer data. `acquire()` normalizes and validates using that consumer schema before even checking whether the provider type exists. Frame size limits do not bound schema evaluation cost. + +**Reproduction:** an authenticated raw client holds a nonexistent remote type with input schema `Type.String({pattern:"^(a+)+$"})` and input `"a".repeat(23) + "!"`. A 24-character input blocked the event loop for approximately **297 ms**. Slightly longer inputs grow exponentially. No valid provider or successful route activation is needed at the targeted host. + +**Fix:** do not execute arbitrary peer validation programs as admission logic. Resolve an authorized provider first, normalize using provider-owned schemas, and validate consumer descriptions against a deliberately bounded schema grammar/complexity budget. Compatibility checking also needs safe handling of patterns/references. Add raw-wire adversarial schema tests, not just malformed-frame tests. Authentication alone is insufficient: a read-only guest is still an untrusted peer. + +### 3. Major — a live consumer remains healthy after its declared dependency fails + +**Location:** `packages/chord/src/object.ts:294–300,333–345`; `self.ts:94–103`; `packages/chord/docs/step-6.md`, “Whole-object failures”. + +Declared references are awaited during activation, but their later lifecycle is not supervised by the owner. Failing the dependency detaches its feeds and marks its references failed; it does not close the dependent object's gate. A derived view can continue exposing its last successful value and accepting unrelated methods. + +**Reproduction:** local `Core` exposes a derive which throws after an input changes; local `View` declares `self.get(Core,{})`, derives a value from it, and offers `ping() => 42`. After the failure: **Core failed; View live; `ping()` returns 42; View still exposes the old value 0**. + +**Fix:** supervise declared dependencies throughout an attempt, not just activation. Define failure/loading transitions, close dependent admission immediately, and reconnect/recompute before reopening it. Preserve stale values only with a visible non-live state. Test post-live dependency failure, retry and link loss, including dependencies not used by any derive. The implementation note currently promises behavior this trace contradicts. + +### 4. Major — ending an object neither promptly rejects its callers nor bounds teardown + +**Location:** `packages/chord/src/object.ts:337–338,367–397,569–580`; design §3 timeout table and §6 lifecycle. + +The method's child scope correctly includes object/attempt cancellation, but the returned promise races the invocation against **only the caller's** signal. `end()` then waits without a deadline for every underlying invocation. Failure cleanup does the same. An uncooperative handler therefore keeps its caller, cleanup and subsequent activation blocked. + +**Reproduction:** a remote method awaits an externally controlled promise and ignores its context. Start it and call `host.close()`. The method remains pending after a 50 ms observation; shutdown also cannot complete until the latch is manually released. The source has no eventual drain deadline, so this is not merely a slow test. + +**Fix:** race the caller-facing result against the linked child signal; separately track and drain the actual handler with the decided per-type bound. After the deadline, report retained work and finish teardown without permitting late authority writes. Test object failure, explicit shutdown and revocation with a handler that never settles, and late handler settlement after cleanup. Native cleanup must not be layered on top of this unbounded wait. + +### 5. Major — `host.root` scopes are not owned or cancelled by the host + +**Location:** `packages/chord/src/host.ts:365–381,430–480`. + +A root links only the supplied caller signal. The host records neither its scope nor its completion. Consequently host shutdown can return while external ingress work is still running with a non-aborted context. + +**Reproduction:** run `host.root(async ctx => { capture(ctx.signal); await latch; })` on an otherwise empty host. After `await host.close()`, **the captured signal is not aborted** and the callback is still waiting. This matters for future HTTP routes and Durable hooks that start work through `root`; not all such work holds an object method invocation. + +**Fix:** give root scopes a host-lifetime cancellation source and track them for bounded cleanup. Keep cancellation of the caller-facing wait distinct from joining arbitrary user code. Add a root-only shutdown test and an ingress callback that observes cancellation while using an external service. + +### 6. Major — one incompatible consumer can fail an otherwise valid shared provider object + +**Location:** `packages/chord/src/contract.ts:268–290`; `object.ts:440–445,505–507,589–593`. + +Coverage allows a consumer to mention a subset of provider variants. Installation then validates the selected provider variant against that consumer. If it is absent, the installation error calls `object.fail()`, poisoning every binding to the object instead of refusing just that consumer. + +**Reproduction:** provider contract has variants `a` and `b`, factory returns `b`; obtain a valid full-contract reference. Bind a same-name/version/input consumer containing only `a`. The new get fails with `invalid/implementation: unknown variant b`, and **the already-valid full reference becomes failed too**. + +**Fix:** decide variant compatibility before installing a binding and distinguish provider-invariant failure from consumer/adapter binding failure. Reject an unsupported consumer with a binding-local mismatch; do not restart a correct provider. Test a stale or deliberately restricted consumer alongside a valid consumer of the same principal/object, including over the wire. + +### 7. Major — member dispatch uses the first variant's signature instead of the active variant + +**Location:** `packages/chord/src/reference.ts:260–275`. + +The proxy selects the first matching member across all variant descriptions and caches a closure containing that descriptor. Contracts permit variants with different method arities, and even different member kinds. TypeScript narrowing therefore promises calls that the runtime misinterprets. + +**Reproduction:** variant `a` declares `run: method(undefined, Type.String())`; variant `b` declares `run: method(Type.String(), Type.String())`; factory returns `b`. `run("hello")` treats `"hello"` as a context and fails with **`invalid/foreign_ctx: ctx must come from Chord`**. + +**Fix:** dispatch from the active shape/variant, including for a previously extracted method function. If variant-dependent kinds/arities are intentionally forbidden instead, enforce and document that at contract definition and reflect it in types; do not silently accept them. Test both variant orders, method/state collisions, loading-to-live admission and extracted functions. + +### 8. Major — released contract bindings and failed-address records have unbounded retention + +**Location:** `packages/chord/src/object.ts:218–224,416–423,427–437`; `host.ts:159,284–287,320–323,351–355,757–760`. + +A local object's binding map never removes a zero-hold binding; detaching even retains member snapshots. A remote peer can create arbitrarily many compatible descriptions by changing an annotation such as `title`, then release each hold. A legitimate reference keeping the object alive keeps all those dead bindings alive too. Separately, failed-address backoffs are deliberately not removed on object/binding disposal, with no TTL or capacity. + +**Reproduction:** keep one valid reference, acquire/release 100 same-name compatible consumer contracts differing only in the input schema's `title`. Inspection of the object's binding map after all releases reports **101 bindings**. Their subscriptions are detached, but their tokens and snapshots are retained. + +**Fix:** evict zero-hold bindings after any explicit linger and keep only bounded retry metadata. Give failed-address backoffs a TTL/cap that still prevents immediate retry storms. Add repeated hold/release tests with unique descriptions and failed addresses, measuring internal retained counts rather than only active holds. Admission quotas are also needed for live holds; a frame-size cap is not a cardinality cap. + +### 9. Major — SQLite delta commits replay the entire existing tail on every write + +**Location:** `packages/chord/src/storage/sqlite/storage.ts:61–84,143–179`; `state.ts:33,42`; `documents/owner.ts:632–651`. + +For each changed document, `commit()` materializes its base and all deltas to construct a temporary `MemoryDocumentStorage` validator. With no `checkpointWhen`—a supported default—N tiny changes perform quadratic cumulative replay, despite the host already possessing validated materialized authority. Even a new checkpoint write first pays for the old tail. + +**Reproduction:** one SQLite document `{n:0}`, then three consecutive batches of 250 tiny delta commits. Batch times were approximately **113 ms, 217 ms, 346 ms**. The increasing work follows directly from the unbounded tail scan, not from document size growth. + +**Fix:** bound physical delta tails by default and separate batch/lifecycle validation from full history replay. At minimum perform bounded backend checkpoints without requiring every plugin author to predict this failure mode. Avoid solving it with a permanent unbounded mirror of all documents. Add a long-lived hot-document benchmark and query/replay-count assertions, including no-op and checkpoint writes. + +### 10. Major — byte validation enumerates one JavaScript property per byte + +**Location:** `packages/chord/src/values.ts:108–111,403–409`; design §§22–23, raw byte transport and approximately 1 MiB chunks. + +`isBytes()` implements the “no extra properties” check with `Reflect.ownKeys(value).length === value.length`. For a typed array this allocates a list containing every index key. Validation repeats at multiple frame and value boundaries, so native byte transport still performs expensive per-byte string-key work on the event loop. + +**Reproduction:** three `isValue(new Uint8Array(1024 * 1024))` checks averaged approximately **102 ms per check** on this host. That is only one validation seam, before packet encoding or relay/provider checks. The clone benchmark motivating zero-copy bytes does not measure this cost. + +**Fix:** treat bytes as opaque leaves with constant-time recognition and define/ignore nontransported attached properties, rather than enumerating indexed elements. Keep dev-only mutation diagnostics separate from the production path. Benchmark end-to-end 1 MiB chunks through WebSocket → relay → IPC, including validation counts, latency and allocation pressure. + +### 11. Major — the playground does not enforce one writer or exit-before-replacement + +**Location:** `apps/chord-playground/server.ts:43–55,61–95`; `worker.ts:28`; p3 functionality map, “one worker per store”. + +The server has no data-directory process lock. Route resolution forks immediately and overwrites `workers[id]`; abort sends SIGTERM but does not make the next resolution await the previous process's exit. The worker opens the session SQLite file without an incarnation/process lock. SQLite serializes individual transactions, not the two hosts' independently cached document authority. + +**Failing scenario:** two accidental playground servers point at the same data directory, or a link disappears before its old worker has actually exited and another resolution starts. Both workers can materialize the same old document and commit valid but conflicting updates. An incarnation in the handshake detects a replacement; it does not fence the old storage writer. Current crash tests explicitly wait for SIGKILL/exit before making the next call, avoiding this case. + +**Fix:** enforce an exclusive server/store lock and serialize replacement through confirmed exit, with bounded termination and a worker lock as the final guard. Keep process supervision in the app/adapter as decided; do not make Chord a process manager. Add duplicate-data-directory startup and link-loss-with-live-old-worker tests before using this playground supervisor as p3's template. + +### 12. Major — staging-time registry composition has no defined answer for activation-dependent entries + +**Location:** design §§13,16–17,21,24; decision log “Registries and HTTP” and “Durable harness inputs”. **Design gap; registry/reload implementation is not present.** + +Factories/staging cannot read a new native handle's `.value`. Registry composition must run during staging for rollback, yet entries may carry natives and disappear when their owner fails. The design does not specify the distinction between candidate declarations and ready runtime entries, nor how recomposition failure after activation is handled. + +**Failing scenario:** an environment/model runtime contributes a native-dependent value; staged composition needs its metadata to detect a duplicate or select a provider. It either reads an unopened handle and rejects a valid reload, or postpones validation until activation and discovers a conflict after the rollback point. If an opener then fails, including its candidate entry exposes a not-ready handle; excluding it changes the composed value after commit. A successful staging composition alone proves neither runtime availability nor a usable required provider. + +**Fix:** make a narrow rule explicit: staging composes static descriptors and lazy entry functions only, without native/document reads; runtime availability is a separate state and required services gate their consumers. Specify atomic publication/removal and errors for post-commit activation failure. Prove startup and reload with a real Models/Environments/Harness graph, an opener failure and a composition conflict before general registry helpers proliferate. + +### 13. Major — pinned Durable phases and sealing a reloaded sandbox conflict for delayed tool entry + +**Location:** decision log “Code mode in p3 drains its sandbox in close”; design §§13,17,24; `packages/durable/src/harness/tool.ts:115–176,413`; `scheduler.ts:1320–1337`; `apps/p2/plugins/codemode/worker.ts:37–55`. **Design integration gap.** + +Durable pins a tool/registry snapshot for a phase, including its `beforeTool` work. The proposed native reload seals the old sandbox, drains scripts already running, then closes it. A phase can hold the old entry without having started its script yet. “New scripts start on the new sandbox through the registry” is not true for that already-pinned entry. + +**Failing scenario:** a code-mode invocation resolves its tool, waits in an approval hook, the code-mode plugin reloads and drains an empty old sandbox, then approval completes. The pinned old tool now tries to enter the sealed/closed sandbox. Waiting for currently running scripts alone cannot make this case safe. + +**Fix:** choose explicit semantics: acquire an execution lease before the phase can wait; retain the old provider until relevant leases finish; or make the old tool entry resolve a compatible current sandbox when execution actually starts. If delayed old invocations should fail, document that weaker behavior and ensure Durable does not replay an unsafe side effect. Add this approval/reload race and the worker-stop/replay counterpart to the sandbox prototype. Do not generalize this into a global code-version lease system unless this concrete boundary needs it. + +### 14. Minor — relocation needs an explicit native-owner pattern, not an assumed reopen + +**Location:** design §§13,29; `apps/p2/src/runtime/worker.ts:55–65`; `apps/p2/plugins/workspace/worker.ts:123–158`; p3 functionality map domains 1,9,10. **Porting/documentation gap, not an impossibility in the API.** + +A native handle never changes during an attempt; changing it requires restarting its owner. A session's stable identity remains the same while cwd/environment changes. A direct port which opens one watcher at `self.open` and assumes it follows the session will therefore keep watching the old location. + +**Failing scenario:** `WorkspaceBackend {conversation}` opens a watch in cwd A; relocation commits cwd B without a code change or failure. The stable reference still names the same live owner and native. Late callbacks from A can continue publishing after the move. + +**Fix:** document and test an owner-owned native manager whose stable value explicitly switches its internal watches with generation fencing; p2's workspace already contains such a pattern. Generation-keyed native owners are another option. Neither requires restoring arbitrary resource versions or adding a generic object-restart API. Prove relocation during open, a failed replacement and late callbacks from the old environment before using this as the prompt/workspace port template. + +### 15. Major — jobs that outlive calls lack an explicit owner-lifetime rule + +**Location:** design §§6,15,20, especially default referenced lifetime and “owned work ... outlives the call”. **Design gap; jobs are not implemented.** + +A referenced object ends after its last reference, and ending aborts its jobs. `job.start()` returns a run independently of the starting call, but the design does not say whether an active/finishing job pins its object or whether the run handle holds a reference. Worker keep-alive is not object keep-alive. + +**Failing scenario:** an HTTP/root call gets a referenced shell or login owner, starts a job and returns its id. The call-scoped reference is released at settlement. With no UI hold yet, the owner ends and aborts the just-started job. There is also a gap before the caller gets a separate `LoginFlow {id}`. A finishing job whose `finish` retries needs a policy after the last viewer leaves too. + +**Fix:** specify whether active and unacknowledged job records pin their owner until settlement, with explicit cancellation/removal and bounded retention for blocked outcomes. Alternatively require these job owners to be host-lifetime and document the resulting address/state organization. Test start-and-immediately-release, disconnect, no viewers, failed finish, plugin removal and persisted crash recovery. Do not add jobs-as-objects merely to avoid deciding this ownership rule. + +## Readiness decisions and scope to cut + +- **Preserve the shared-core/per-principal-view split**, but fix finding 1 first. Principal identity must survive remote method dispatch; internal local ownership and explicit attribution remain separate concepts. +- **Preserve per-host rollback**, but only for load/declaration/composition failures. Do not quietly extend it to irreversible native opens or migration effects. Resolve finding 12 with a small staged-descriptor/ready-provider example rather than a new distributed transaction protocol. +- **Mixed revisions are not demonstrated.** The committed suite has no rollout implementation to validate. Before calling p3 ready, exercise server commit + worker rejection, a joining worker, a client behind the provider, and the consumer itself being behind/rejected. The decision log already acknowledges own-status checks, serialized installs and complete-code-set digests; those remain required, not optional polish. +- **Durable should remain the resumable workflow engine.** Chord jobs should stay a small owner-local intent/outcome mechanism. Do not duplicate Durable's scheduler, task graph or crash replay. Solve finding 15 before building generic flow/job addressing. +- **First real plugin proof should be Models/Environments → Harness → one tool**, with an approval pause and a code-mode reload. Notes plus a counter does not exercise registry snapshots, native restart, prompt readiness or delayed tool entry. +- **Presence and optional providers need explicit contracts before UI ports.** The decision log still leaves link-drop lingering, origin counts and optional references open. Test two tabs, one disconnect, reconnect and explicit navigation before deriving guest presence, worker idleness or login cancellation from them. Do not infer “user left” from a hot-reload unmount. +- **Keep p3 isolated and start empty as decided.** Do not port legacy catalog/cookie migrations, old attribution parsing or cross-plugin document reach-through. Core-owned creation-time conversation policy is the right place to preserve read-only/follows-cwd rules when a plugin is absent; its API needs a concrete data schema, not an opaque Durable callback. +- **Defer broad optional HTTP/channel abstractions, general middleware phases and a Durable Object execution profile** until the kernel findings and one native/registry/Durable proof are resolved. Native byte channels still need their own explicit connection cancellation and revocation path; a successful Chord `revoke` test alone does not prove an already-open terminal channel is closed. +- **Cut repeated work before adding caches:** remove per-byte key enumeration and full-history validation per delta. Bound existing binding/backoff retention before adding more lifecycle registries. These are simpler fixes than compensating with another batching or caching layer. + +## Minimum additional regression matrix + +The passing tests establish the implemented happy paths, not p3 readiness. Add, at minimum: authenticated remote method principal and nested acquisition; consumer-selected pathological schemas; post-live dependency failure/recovery; uncooperative teardown and root shutdown; incompatible consumer isolation; variant-dependent member dispatch; binding/backoff retention bounds; hot SQLite documents and end-to-end byte chunks; writer fencing on live-worker link loss. Then add registry/native activation failure, mixed-revision rollout, delayed Durable tool entry across sandbox reload, session relocation and job owner lifetime when those features land. + +Documentation cleanup is secondary: `kernel.md`'s old shared-import list omits the subsequently approved `../sqlite` driver layer; the decision log explicitly permits it. That import is not an architectural violation. Conversely, step-6's claim that dependent views fail on a broken owner should not be treated as implemented until finding 3's trace passes. diff --git a/apps/p2/docs/p3-reviews/v2-review-fable.md b/apps/p2/docs/p3-reviews/v2-review-fable.md new file mode 100644 index 000000000..edca2a88a --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review-fable.md @@ -0,0 +1,314 @@ +# Chord v2 review (fable) + +Reviewed at `e5864dd82` (branch `private`), kernel through step 6 part 1 (derives). Uncommitted natives work was +ignored. Sources: `packages/chord/docs/{design,kernel,step-*}.md`, `packages/chord/docs/decisions.md`, +`apps/p2/docs/chord-next-walkthrough.md`, `packages/chord/src/**`, `packages/chord/test/**`, +`apps/chord-playground/**`, with today's Chord, p2 plugins, Durable and `p3-functions/` for context. The 412 kernel +tests pass at HEAD. Probes were run from a clean worktree (`/tmp/chord-head/packages/chord/test/probe/*.test.ts`, +throwaway); their output is quoted where it matters. File:line references are to HEAD. + +Severity: **blocker** (must be settled before p3 plugins are written on it), **major** (will bite a real plugin or +leak/stall in production), **minor** (hardening, drift, perf). + +--- + +## A. Findings + +### 1. blocker: every remote method runs with the host's principal; `ctx.principal` and `ctx.get` escalate to host authority + +Design §5 table: "`ctx.principal` in methods: the caller". §8: "`principal` (remote only)". §23: "`ctx.principal` is +the request's principal, so `ctx.get(Notes, …)` gives that principal's view". Walkthrough concept 12 says the same. +`MethodImplementation` types `ctx` as `Ctx

`, so authors will read `ctx.principal` believing it is the caller. + +Code: `object.ts:536-547` (`LocalBinding.call`) builds the handler's scope with `this.host.principal!` for both +local and remote; `connection.ts:1001` (`#call`) runs incoming calls under `host.root()` with no principal, so the +link-stamped principal (`served.request.principal`) never reaches the method. `step-3.md:35` ("Object methods retain +execution-as-host semantics") and `test/host-smoke.test.ts:45` / `requests.test.ts:104` lock the opposite of +the design in. + +Probe (`principal.test.ts`): guest `{kind:"user",id:"guest-1"}` holds `Notes {S1}` through server → worker and calls +`ctxPrincipal()`; the handler sees `{"factory":{"kind":"user","id":"guest-1"},"ctx":{"kind":"worker","id":"S1"}}`. A +method that does `const audit = await ctx.get(Audit, { session })` gets the *worker's* variant of `Audit` +(`{"kind":"worker"}` returned by `audit.who()`). Same host: `host.root(fn, { principal: user })` → +`ctx.get(T).who(ctx)` → `{kind:"server"}`. + +Why it is a blocker and not a doc nit: the "shared core, per-principal views" pattern only stays safe as long as a +view never delegates to another *remote* view. The first p3 case that does (Chat's `send` writing through +`Workspace`, an HTTP route doing `ctx.get(Notes, …)`, a tool approval hook) silently acts as the worker/server. And +there is no plugin-reachable way to act as the caller: `FacetApi.root` has no `principal` option (`facet.ts:13`), +`ctx.get` has none, only app code holding the `createHost` result can call `host.get(T, i, { principal })`. + +Fix (pick one, then align design, walkthrough, step-3 and tests): +(a) `ctx.principal` = the link-stamped caller (data only), `ctx.get` keeps acting as the host (explicit in the + docs), plus `ctx.as(principal)`/`root(fn, { principal })` on `FacetApi` for the HTTP and hook cases; or +(b) keep execution-as-host everywhere, remove `principal` from `Ctx`, make §5/§8/§23 say so, and give HTTP routes + `req.principal` plus a principal-scoped `root`. +Either way: thread `request.principal` from `connection.ts:#call` into `LocalBinding.call` and add a test that a +relayed guest call observes the guest. + +### 2. major: references do not re-hold after a routed link drop; passive viewers stay `loading` forever + +Design §1 goal 5 "References re-hold by themselves", §3 "after a drop the consumer re-holds every hold over the new +link", §3 route "Chord calls `resolve` ... again after a drop", walkthrough concept 8 model answer ("the values stay +readable with `loading`; ... when the new incarnation is up, the reference re-holds"). + +Code: when a routed child link closes, `connection.ts:1135` marks relayed holds `detached` and sends `status loading`; +`remote.ts:134` (`detached`) puts the binding in `idle` and `loading`. Nothing schedules a new `resolve`: only a new +hold, a call (`#call` on `detached` → `#open`) or an explicit `retry` frame reaches `connectionFor`. Consumer-side +`reconnect()` exists only for the client's own parent link (`host.ts:connect`). + +Probe (`recovery.test.ts`): client holds `Counter {S1}` (state only); worker host closed; after 500 ms the viewer is +still `loading` and `resolves === 1`. A pre-existing `server.get(Counter {S1})` (the shape of every "published +projection" or server facet dependency into a worker) is likewise `loading` with `resolves === 1`. A fresh +`server.hold` afterwards triggers `resolve` and revives everyone (the playground kill test only passes because the +owner clicks `add`). + +Impact for p3: a tab showing a conversation whose worker crashed, a session list projection, the subagents parent +holding a child's conversation, the ports plugin watching a catalog: all stall until a user action. Fix: on +`connectionClosed` for a routed target, schedule a re-resolve with the per-address backoff while any served +`detached` request or idle binding with refs exists for that address (the trigger model the docs already describe); +or make the design say honestly that views do not recover without a call and give the UI a `ref.retry()` button. + +### 3. major: no drain deadline anywhere: a handler that ignores `ctx.signal` hangs object end, `revoke` and `host.close()` + +Design §3 timeouts: "call drain: waiting for an ending object's own call handlers (callers are rejected at once), = +`closeTimeout`". Code: `object.ts:385` (`end`) and `object.ts:300` (`fail`) await `Promise.allSettled([...calls])` +with no bound; `host.ts:#finishClose` awaits every `object.end()`; `host.revoke` awaits them too. + +Probe (`revoke.test.ts`): guest calls `hang()` (never resolves, ignores the signal); `server.revoke` resolves (the +server owns no matching object) but `cluster.close()` never completes (vitest timeout); the worker's own `revoke` +promise dangles. The playground's worker `stop()` would never disconnect; only the server's 2 s SIGKILL saves it. +p3 handlers (SSH exec, model streams, PTY reads) routinely take seconds to honor a signal. Fix: bounded drain with +the host default, then report and proceed; `revoke` must not await drains at all. + +### 4. major: a facet failure is permanent and poisons every object it serves + +`facet.ts:83` (`fail`) sets `failed`, aborts the controller and ends the self; nothing ever calls `activate()` again +(only from `declaring`). `host.ts:#failFacet` fails served objects, which then retry with backoff forever: +`object.ts:#activate` awaits `service.activation()` = `facet.ready`, a permanently rejected gate. + +Probe (`facet.test.ts`): worker facet does `self.get(Catalog)` (server type); the parent link is not up within +`readyTimeout` → `load()` rejects `ready_timeout`; afterwards `worker.hold(Core, {})` reports +`failed: facet:probe.catalog was not ready after 200 ms` indefinitely. In p3 a worker that starts while the server is +busy (or the relay is reconnecting) loses its whole plugin until the process restarts. Design §6 gives objects +transient backoff; facets need the same (or a facet must not fail on dependency timeouts at all: keep it `loading`, +objects park). + +### 5. major: full-schema validation of large state on every change, at the provider per binding and at every consumer + +`object.ts:472,478`: every event of every exposed state member is validated against the consumer contract schema +(whole value), and `remote.ts:prepare` validates the whole applied value again at every consumer. + +Probe (`validate.test.ts`, `perf.test.ts`): `assertValid` on a 500 KB chat-shaped value: 8.2 ms (vs 1.1 ms +`cloneTrusted`, 0.5 ms `JSON.stringify`, 0.06 ms for the tracker change). One-character append with 0 remote viewers: +7.9 ms; 1 viewer: 23 ms; 3 viewers: 54 ms (in-process viewers, so consumer-side validation is included; a real +worker still pays the 8 ms per change plus the diff work per derive). A streaming turn at 20 deltas/s on a 500 KB +view spends ~160 ms/s of the worker's loop on validation of its own authoritative state. Today's Chord validates at +the consumer only (`src/connection.ts:1318`); v2 added the provider side. Fix: validate once per publication at the +provider (per object, not per binding; members shared by several contracts), validate ops by path or skip +consumer-side validation for same-origin trusted hops, and cache `isValue` for immutable values (`WeakSet`). + +### 6. major: reload semantics for native objects make terminals and the harness un-reloadable in practice + +Design §21: native objects of a reloaded plugin restart ("running work ends `aborted`, natives close"); remedy: "split +that native into its own plugin". But §13/§14 put the behaviour around a native (routines) in its owner. For +terminals (`p3-functions` #11) the owner of a PTY is also the byte pump; any change to the pump kills every shell. For +the harness (§24) a change in the plugin that opens Durable aborts every running agent task. The "split" remedy +yields a plugin per native with routines that may never change, which is the p2 complexity the design wanted gone. +Also undefined: §21 "pure objects swapped in place ... running calls finish on old code" vs the kernel's handle model: +`self.end()` deactivates document handles (`self.ts:131`, `owner.ts:112`, `HANDLES.active=false`), so an in-flight +`tx.draft(oldHandle)` throws `gone` the moment the new factory's self replaces the old one. Decide: either natives may +be retained across a reload when the new factory declares the same native (opt-in, e.g. keyed `self.open` with +`keep`), or state that terminals/harness are "restart only" plugins; and define what happens to running transactions +on swap (keep old handles valid until their calls settle). + +### 7. major: IPC links have no flow control at all + +`link.ts:133` `IpcLink.transmit` returns `undefined` → `Connection.send` counts size 0 → `#full` is never true +(step-3 "native unsized IPC never waits for byte credits"). Node's `child.send` buffers unboundedly when the pipe is +full. A worker publishing large state (chat view deltas, logs, status) to a server whose loop is busy grows the +worker's heap without bound and the server's inbox likewise; no reset plan ever triggers. Design §3 "flow control per +link (carried over)" is not true for the only server↔worker transport. Fix: size IPC frames (cheap: reuse +`frameSize`, or count messages) and apply the same window. + +### 8. major: `accepts` adapters are not the pure functions the design describes; authors must hand-build Readables + +Design §5/§27: `accepts: { [old]: adapter }` maps provider implementation to the old view. Code: +`object.ts:443` `implementationMembers(this.contract, view, false)` requires every state member of the adapted view +to be an object with `subscribe` and `events`; `test/adapted-state.test.ts:17` defines its own `mapped()` helper +to satisfy that; Chord ships none. The first real version bump in p3 will fail with `invalid/implementation: doc must +be a readable`, or worse, authors will return `{ value }` objects that pass nowhere. Fix: adapters map *values* per +member (`{ state: { doc: (v2) => v1 }, methods: { … } }`), Chord derives; keep per-binding diffing internal. + +### 9. major: revocation ends views but not the work a revoked guest started in trusted cores + +Design §4 and §13: revocation ends objects whose principal matches; "Chord aborts only the object's own calls and +jobs". Calls chain (the view's call scope aborts when the view ends, `object.ts:536-547`), so a guest's *call* into a +local core stops. Jobs, flows (`LoginFlow` as a keyed document, Questions), lingering core state keyed by guest id, +and provider-side `linger` objects do not: the core has no principal and no hook. For sharing (`p3-functions` #12) +"revoke live access" must also end a removed guest's running shell job, pending question, and queued sends. The +design needs either a `host.onRevoke(r)`/registry hook for cores or a rule that cores key guest-initiated work by a +revocation label and reconcile from `host.references`. + +### 10. major: relayed holds that fail permanently retry forever and hide the cause + +`connection.ts:537` (`#open` catch) treats every failure, including `forbidden` from `route.authorize`, as transient +with a 1–60 s backoff for as long as the client keeps the hold (and the failed reference "stays alive" by design §7). +Calls on a `failed` served entry reject with `gone: unknown or failed hold` (`connection.ts:1027`), not with the +cause (§7: "reject at once with the cause"). Each denied hold on the server costs a timer and an `authorize` call per +retry per client. Fix: map `forbidden`/`invalid/placement`/`contract_mismatch` to the design's `broken` (no automatic +retry) and return the cause on calls. + +### 11. major: a worker start is killed when its first requester leaves + +`host.ts:720`: when the last pending requester of a resolving route goes away, the resolver signal aborts; the +playground kills the child (`server.ts:77`). In p3 a worker takes seconds (Durable open, environment probe); opening a +session and navigating away within that window kills it mid-start, and the next visit starts another. Repeat on a +flaky phone and the session never comes up; a kill during Durable's open is also where storage corruption happens. +Combined with 12: nothing keeps a half-started worker alive. Fix: let a started worker finish and register as +`live`; abort only resolvers that have not produced a link. + +### 12. major: consumer-side linger cannot do what §19 says; keep-alive inputs do not exist yet + +§19: "`ref.release({ linger: true })`: its hold stays, marked lingering (a closed tab, 2 minutes); it keeps the worker +alive". A closed tab closes its socket; `remote.ts:136/179` and `connection.ts:#closed` discard every lingering lease +on link drop (documented in step-3). So only the *provider-side* `ServeOptions.linger` can keep anything alive after +a tab closes, and it is invisible to presence. The decision log admits "a link drop is still a departure" but the +design text and walkthrough still sell the closed-tab case. Meanwhile `host.references`, activity counters and any +idle computation (decision #1) are absent, so the playground cannot even stop a worker. State the truth in §19 and +decide #1 before p3's server exists. + +### 13. minor: backoff maps leak per failing address + +`host.ts:354` and `forgetBinding`: `#backoffs` entries are deleted only when no failure is recorded. Every +per-principal remote object or remote binding that ever failed leaves an `AddressBackoff` forever (guest tabs +hitting a throwing factory, denied addresses). Bound it (expire after the cap delay) or store the failure on the +object table entry. + +### 14. minor: unbounded per-link resources for an authenticated client + +`connection.ts:987` `maxCalls` defaults to `Infinity`; hold ids are unbounded; first views bypass the window +(`connection.ts:579,939`); `call`/`result` frames never wait for credits. One guest tab can hold thousands of +addresses (each activating an object and a `ParkedCalls`) or queue megabytes of first snapshots. Defaults for +`maxCalls`, max holds per link and total unacked bytes for non-state frames are cheap. + +### 15. minor: deeply nested frames crash IPC peers + +`values.ts:104` `isValue` is recursive; a 200k-deep array makes `isFrame` throw `RangeError` (probe `deep.mjs`). On +WebSocket links the `#message` chain catches it and closes the link; on IPC the exception escapes the `message` +listener and kills the process. IPC peers are trusted today, but a bug producing deep JSON would take the server down +with a worker. Make `isValue` iterative or depth-limited; validate before `deliver`'s `bye` check. + +### 16. minor: keyed reads take the exclusive line + +`owner.ts:132` (`read`) acquires the owner line `exclusive` for `handle.get(key)` and `keys()`. Chat history paging +(the design's "paged method" for large collections) then serializes every reader behind every writer of the same +object. Reads should take `shared`; only transactions need exclusive. + +### 17. minor: a server storage failure leaves a zombie process + +`host.ts:189` `storageFailed` → `host.close()`. The playground worker's `onError` stops the process; the server has +no handler: the WebSocket listener stays up, the next `host.accept` throws `gone` inside the `connection` handler, +which is an uncaught exception. Design §10: "the host reports it and restarts" names no owner for the restart. Either +Chord exits with a distinct code or the app must be told to supervise. + +### 18. minor: documentation drift that will mislead plugin authors + +- Walkthrough "Open points at a glance" still lists `close` with a reason and diffs for adapted bindings as open; + both are decided (log) and the latter implemented. Concept 14's Open block contradicts design §13. +- `RefState.failed` is `{ cause: FailureCause; retryAt: number | undefined }` in design §7, `{ cause: ChordError; + retryAt: number }` in walkthrough concept 8 and code. +- `step-3.md:35` and `kernel.md` describe execution-as-host; design §5/§23 say caller (finding 1). +- §19 closed-tab linger (finding 12). +- Design §3 timeouts table promises per-`get`/per-type overrides (`readyTimeout`, park deadline); `GetOptions` and + `ServeOptions` have none; one `readyTimeout` serves activation waits, parked calls and derive hydration. +- `principalKey` must be the same function on every host (the provider keys objects, the consumer keys bindings and + `revoke` matches by it); the design says this for `revokedBy` only. + +### 19. minor: state frames are encoded up to three times on WebSocket links + +`connection.ts:#writeState` calls `link.check(frame)` (`frameSize` = full `plan` + `JSON.stringify`), then `send` +(`encodePortable` = plan + stringify again), and queued frames are sized again for `waiting.bytes`. `plan` also +allocates a path array per node. For a 1 MiB snapshot that is three serializations per send. + +### 20. minor: remote dependency cycles spin forever + +`host.ts:acquireOwned` detects cycles only among local bindings. A worker object holding a server object that holds +the worker object waits `readyTimeout` on both sides, fails transient, retries, and repeats every 1–60 s with the +message "dependency cycle?". Detect at least the two-hop case by carrying the activation chain in the hold frame, or +mark the second timeout `broken`. + +### 21. minor: transaction/document footguns to document or fence + +- `tx.get`/`handle.initial()` return the live readable value (same object as `handle.value`); outside dev mode a + caller mutating it corrupts the authority (`owner.ts:get`). +- `keys()` for in-memory keyed docs scans every entry on the host (`store.ts:memoryKeys`). +- A read-only transaction on a document with a newer definition writes a base to storage (`owner.ts:finish`, + `base = storedVersion !== version`); fine, but a read in a hot path after a version bump turns into a write storm + until every document is touched once. Mention it in §9. + +--- + +## B. Does the design hold together? + +| Area | Verdict | +| --- | --- | +| Hosts and routing | Sound shape; `connectionFor` always goes to the parent when one exists, so workers reach siblings only via the server (fine). Recovery after a child drop is pull-only (finding 2) and start/abort is caller-coupled (finding 11). | +| Principals and revocation | Stamping and forwarding are correct (clients cannot forward; only the parent is trusted; `revoke` frames only from the parent). Execution principal contradicts the design (finding 1); revocation does not reach work in cores (finding 9); `principalKey` consistency is unstated. | +| References, failure, backoff | Transient/broken split is not in HEAD; everything retries. Shared per-address backoff is right. Failed facets never retry (finding 4). Dependency timeouts mark the dependent failed instead of parking (design says park then transient; code: `awaitReady` 60 s then `fail`), which is consistent with the log but makes a slow server fail every worker object once. | +| Documents and transactions | The strongest part. Lines per owner, facet line second, prepare-before-storage, adopt-after-commit, one batch per link, migrate on access, tags, eviction: coherent and well tested (conformance suite, fences, versions, retention). Gaps: exclusive reads (16), undefined behaviour of in-flight transactions on reload (6), the "hang, not error" rule for `tx` misuse. | +| Natives and close reasons | Not in HEAD. The design is clear except the reload consequence (finding 6). `closeTimeout` must also bound call drains (finding 3). | +| Registries with compose in staging | Reasonable; `compose` in staging is the one place a plugin conflict can roll a host back, good. Undefined: what consumers see between commit and the first `compose` on host start (an empty table?), and how a `lifetime:"host"` object waits for a required contribution without a Chord API (§24 "wait for the required contributions"). | +| Reload with per-host rollback | Staging with a recording `self` is plausible; the kernel's `SelfImpl` per attempt supports it. Undefined: in-flight transactions and `ctx.get` references of running calls when the self is swapped (6); routines serialized per name need a routine identity that survives a rename. | +| Rollout | Not in HEAD; the mismatch table is fine. Open #12(a) matters: a client ahead of the server after a page reload parks every call for 60 s instead of reloading. | +| Presence | Finding 12. | +| Jobs | Sandwich is right. Jobs can exist only in stable identities if they persist; a guest's job must therefore live in a core with attribution as data, and revocation cannot find it (finding 9). "Jobs as objects" (#9) would fix both the flow-address question and revocation (an object has a principal) and should be decided before login/shell are ported. | + +## C. What will break when the real p3 plugins are built + +- **Chat with Durable:** harness restarts on any reload of its plugin (6); conversation view validation cost per delta + and per viewer (5); tool approval needs a principal-scoped `root` that plugins cannot reach (1); a crashed worker's + open chat views stall (2). +- **Workspace:** file tree as state is fine; large listings as paged methods serialize behind writes (16); "workspace + not for guest read" is decided in the view's factory, correct, but any workspace method that delegates to another + remote owner escalates (1). +- **Terminals:** channels are not in the kernel yet; the PTY owner cannot be reloaded without killing shells (6); + IPC backpressure for terminal state (7). +- **Sharing:** revocation of links/objects works; guest-started jobs and flows survive (9); grant revision bumps + create new objects and orphan nothing (good, persist is refused for remote identities). Guest per-user persisted + state (name, read marks, drafts) needs a server core keyed by grant id with the view passing the id as data. +- **Subagents:** worker→worker through the server as the worker principal is fine; a parent's hold into a child's + worker never recovers after the child restarts (2); `authorize(workerA, sessionB)` needs the catalog's parent/child + relation, which is app policy, fine. +- **Session list / status "without starting a worker":** every hold of a worker-placed type resolves (starts) the + worker; there is no read-without-start primitive; decision #19 (published projections) must come before the + sessions plugin, and finding 2 applies to the server-side copies. + +## D. What could be cut or simplified + +- Coordinated reset collection across relays (`connection.ts` `#collectReset`, `#resyncs`, `#resetRequests`, + shared-response matching, ~300 lines of the riskiest state in the kernel). A relay that overflows can drop the + hold's feeds and re-hold with `again: true`; the provider already answers with a fresh `view`. Per hold, no + cross-link coordination. +- Per-binding diffing for adapted bindings (`object.ts:install` adapter branch, `diff.ts` byte fallback): replace by + value adapters that Chord wraps as derives (finding 8). +- JSONL storage ("dev only, grows forever") and `MemoryDocumentStorage.prepareCommit/snapshot` (built for a removed + Durable adapter). +- Consumer-side linger cache (`#lingering`, cap 4, `presence` frames): provider-side `linger` plus `host.references` + cover presence and keep-alive; the client-side variant cannot survive the case it was designed for (12). +- `accepts` on host values (versioned start data is a schema bump in the app's start message). +- Dev-mode freezing of local arguments/results: it catches nothing for bytes, and `tx.get` already hands out live + authority values unfrozen outside dev. +- The Questions service as a second input pattern; keep flows (state + `answer`) only until a case needs quorum. + +## E. Under-tested + +- No test asserts the caller's principal inside a method (tests assert the host's). +- Recovery after a routed drop is tested only through a call that re-resolves; no test for a state-only viewer or a + pre-existing server-side hold. +- No test for `host.close()`/`revoke` with a handler that ignores its signal (finding 3). +- No test for facet recovery after a failed activation (there is none to test). +- No IPC backpressure test; no test that a 16 MiB state over IPC does not grow memory. +- No multi-viewer cost test; no benchmark for state validation per change. +- `accepts` adapters across a relay and across a worker restart. +- Revocation while a target is `resolving`, and revocation of a principal whose relay hold is `pending`. diff --git a/apps/p2/docs/p3-reviews/v2-review-summary.md b/apps/p2/docs/p3-reviews/v2-review-summary.md new file mode 100644 index 000000000..0edd3d3fd --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review-summary.md @@ -0,0 +1,71 @@ +# Chord v2 review round 1: consolidated + +**Status:** K list confirmed with changes to K7, K9, K16, K17, K21 and D11 decided (see the decision log, "Review +round 1"). Remaining D items are discussed one at a time. + +Sources: `v2-review-astra.md` (A), `v2-review-fable.md` (F), both on committed HEAD `e5864dd82` (before the natives +step), plus the implementation subagent's open questions (Q). Status per item: **fix** (the design already decides it; +a kernel bug), **decide** (needs the owner), **cut?** (a reviewer proposes removing something). + +## Kernel fixes (the design already decides; bugs or missing pieces) + +| # | Sev. | Finding | Source | Fix | +| --- | --- | --- | --- | --- | +| K1 | blocker | remote methods run as the host principal; `ctx.get` escalates | A1, F1 | caller's principal in remote method `ctx` and its `ctx.get` (design §5) | +| K2 | blocker | an incoming hold executes the consumer's schemas (regex DoS, ~300 ms per 24 chars, exponential) | A2 | normalize/validate with the provider's schemas; consumer descriptions only checked within a bounded grammar | +| K3 | major | a dependency failing after `live` leaves the dependent live with stale values | A3 | supervise declared dependencies for the whole attempt | +| K4 | major | no drain deadline: an uncooperative handler hangs object end, `revoke`, `host.close` | A4, F3 | callers rejected at once; drain bounded by `closeTimeout`; `revoke` does not await drains (check what step 6 already did) | +| K5 | major | `host.root` scopes are not cancelled by `host.close` | A5 | host-owned cancellation and bounded tracking | +| K6 | major | one incompatible consumer fails the shared provider for everyone | A6 | binding-local mismatch | +| K7 | major | method dispatch uses the first variant's signature | A7 | dispatch by the active variant (or forbid differing signatures: decide) | +| K8 | major | zero-hold bindings and failed-address backoffs are never removed | A8, F13 | evict bindings; TTL/cap for backoffs | +| K9 | major | SQLite replays the whole delta tail on every commit (quadratic) | A9 | default checkpointing; validation without full replay | +| K10 | major | byte validation enumerates every index (1 MiB: ~100 ms) | A10 | constant-time `Uint8Array` recognition | +| K11 | major | references do not re-hold after a routed link drop; passive viewers stay `loading` | F2 | schedule re-resolve with the per-address backoff while holds exist (design §3) | +| K12 | major | IPC links have no flow control (frames unsized) | A note, F7 | size IPC frames, apply the window | +| K13 | major | a facet failure is permanent and poisons its objects | F4, Q2 | facets retry like objects (transient backoff) | +| K14 | major | permanent relay failures (`forbidden`, `invalid/placement`, `contract_mismatch`) retry forever; calls lose the cause | F10 | classify as `broken`; calls reject with the cause | +| K15 | major | a worker start is aborted when its first requester leaves | F11 | a resolve that produced a link finishes; abort only resolvers without a link | +| K16 | minor | unbounded per-link resources (calls, holds, first snapshots) | F14 | defaults for max calls, holds per link, unacked bytes | +| K17 | minor | deeply nested frames crash IPC peers (recursive `isValue`) | F15 | iterative or depth-limited | +| K18 | minor | keyed reads take the exclusive line | F16 | reads shared, transactions exclusive | +| K19 | minor | state frames encoded up to three times on WebSocket | F19 | encode once | +| K20 | minor | remote dependency cycles spin forever | F20 | detect two-hop cycles or mark the second timeout `broken` | +| K21 | minor | footguns: `tx.get` returns live authority values; in-memory `keys()` scans all; read-only tx after a version bump writes bases | F21 | freeze or copy; index; document | +| K22 | major | playground: no data-directory or worker lock; a replacement can start before the old worker exited | A11 | exclusive server lock, worker lock, replacement after confirmed exit | +| K23 | minor | documentation drift (walkthrough open points, `RefState` shape, execution-as-host in step docs, linger text, per-get overrides not implemented, `principalKey` must be identical on all hosts) | F18 | fix docs | + +## Decisions for the owner + +| # | Topic | Problem | Options | +| --- | --- | --- | --- | +| D1 | registry `compose` vs natives | staging cannot read natives (not open yet), entries may carry natives | (a) staging composes static descriptors plus lazy functions; runtime availability is a separate state that gates required consumers (A12) | +| D2 | code mode reload vs pinned Durable phase | a phase pins the old tool entry, waits in an approval hook; the old sandbox is sealed and closed; the delayed start hits a closed sandbox | (a) execution lease before the phase can wait; (b) keep the old sandbox until leases end; (c) the old entry uses the current sandbox at start (A13) | +| D3 | natives that must follow relocation | a native never changes; a cwd change leaves a watcher on the old path | (a) pattern: a native manager switches its internal watches with generation fencing; (b) generation-keyed native owners (A14) | +| D4 | jobs and their owner's lifetime | start a job, release the reference: the owner ends and aborts the job | (a) running and unacknowledged jobs pin their owner; (b) job owners must be host-lifetime; (c) jobs as objects (open #9) (A15, F jobs verdict) | +| D5 | reloading native objects | terminals (PTY owner = byte pump) and the harness restart on any change of their plugin, killing shells and agent tasks; in-flight transactions of a swapped pure object throw `gone` | (a) opt-in keep of a native across reload when the new factory declares the same one; (b) such plugins are "restart only"; plus: old handles stay valid until their calls settle (F6) | +| D6 | `accepts` adapters | authors must hand-build readables for adapted state | (a) adapters map values per member; Chord wraps them as derives (F8) | +| D7 | revocation and work in cores | a revoked guest's jobs, flows and queued work in principal-free cores survive | (a) a revocation hook for cores (`host.onRevoke` or a registry); (b) cores key guest work by a revocation label and reconcile (F9) | +| D8 | lingering and keep-alive | consumer-side linger is dropped with the link, so it cannot keep anything alive after a tab closes; keep-alive (#1) undecided | (a) cut consumer-side linger, keep provider-side `linger` plus `host.references`; decide #1 (F12) | +| D9 | validation cost of large state | whole-value validation per change at the provider per binding and at every consumer (500 KB: 8 ms per change, 54 ms with 3 viewers) | (a) validate once per publication at the provider; (b) validate ops by path; (c) skip consumer-side validation on trusted hops (F5) | +| D10 | storage failure on the server | the host closes, the process lives on as a zombie | (a) Chord exits with a distinct code; (b) the app supervises via `onError` (F17) | +| D11 | method signatures across variants | variants may declare different arities for one name | (a) dispatch by the active variant; (b) forbid it in contracts (A7) | +| D12 | `retryOn` sources | references are released while failed, so a reference-member source ends | (a) only attempt-independent sources; (b) a hidden hold (Q1) | +| D13 | per-type park deadline for remote consumers | per-type override lives on the provider; remote consumers park on their own host | (a) per-`get` `{ parkTimeout }`; (b) a hint in the contract (Q3) | +| D14 | published projections | every hold on a worker-placed type starts the worker; no read-without-start | decide before the sessions plugin (design #19) | + +## Proposed cuts (F section D, A "scope to cut") + +Cross-relay reset coordination (~300 lines); per-binding diffing for adapted bindings (if D6 a); JSONL storage; +`MemoryDocumentStorage.prepareCommit/snapshot` (built for the removed Durable adapter); consumer-side linger cache (D8); +`accepts` on host values; dev-mode freezing of local values; the Questions service (keep flows); defer the HTTP and +channel components, middleware phases and the Durable Object profile until the kernel fixes and one real +harness/native/registry proof are done. + +## Reviewers' strategic advice + +- First real proof: models and environments → harness → one tool, with an approval pause and a code-mode reload + (instead of more "notes plus a counter"). +- Rollout with mixed revisions is undemonstrated; test server commit plus worker rejection, a joining worker, a + client ahead of the server. +- Durable stays the resumable workflow engine; Chord jobs stay small. diff --git a/apps/p2/docs/p3-reviews/v2-review2-astra.md b/apps/p2/docs/p3-reviews/v2-review2-astra.md new file mode 100644 index 000000000..3b903b9de --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review2-astra.md @@ -0,0 +1,30 @@ +# Chord v2 review round 2 (design): Astra + +Reviewed `packages/chord/docs/design.md` at `92ecf063b` against the decision log and `v2-cost-model.md`. +Proposals only; the owner decides. Kept verbatim from the reviewer's answer. + +Reviewed `92ecf063b`, the full decision log, and cost model. + +1. **Blocker — §§3, 22: opaque forwarding lacks a safe routing unit.** One worker→server commit can contain holds belonging to different clients. Forwarding that opaque body to both leaks the other client’s state; splitting it requires a defined segmentation format. Native IPC→WebSocket also needs an encoding location. **Proposal:** specify independently routable opaque segments, authenticated routing metadata, and atomic commit assembly at each destination. + +2. **Major — §5: contract-cache poisoning is not excluded.** The worker’s parent link multiplexes different principals. If a guest supplies an incorrect description under a legitimate fingerprint, caching it per `(link, fingerprint)` can affect later owner bindings. **Proposal:** require recomputing and verifying the description’s fingerprint before caching; malformed descriptions must fail the originating binding, not the shared parent link. + +3. **Major — §§3, 22: compressed-input limits are undefined.** An authenticated client can send a compressed frame below the encoded-byte limit that expands enormously before provider schema validation. Opaque relaying can carry this into IPC, whose limit is unrestricted. **Proposal:** separately bound compressed size, decoded size and framing metadata; enforce expansion limits during decoding and preserve them across relays. This requires no state-validation walk. + +4. **Major — §§6, 17, 21, 24: the harness start barrier is circular or premature.** The barrier includes every host-lifetime object, including Harness, but Harness activation itself resumes Durable. Waiting for the complete barrier inside its opener waits for itself; resuming earlier can run persisted work before a tool contributor depending on Harness becomes live. **Proposal:** make Harness live while paused, then resume through an explicit post-activation coordinator. + +5. **Major — §§17, 21, 24: approval can resume during the sandbox replacement gap.** Restart removes the old contributor immediately; draining its sandbox can delay the replacement. A previously pinned tool’s `execute()` then looks up the “current” sandbox and finds none. Registry access has no reference-style parking guarantee. **Proposal:** specify a cancellation-aware readiness gate for the current sandbox, with distinct behavior for replacement, failure and plugin removal. + +6. **Major — §§9, 21: old-call guarantees conflict with document migration.** A v1 method awaits external work; reload migrates its document to v2; the method resumes and uses its supposedly still-valid v1 handle. Section 9 requires that access to throw, while §21 promises old calls and transactions keep working. **Proposal:** define the schema-changing exception: defer migration for affected old calls, or explicitly terminate those calls before upgrading. + +7. **Major — §§5, 27: adapted-argument validation order is missing.** V1 accepts `{text}`, V2 accepts `{body}`, and its adapter renames the field. Stripping or validating against V2 before adaptation rejects valid V1 calls; using the consumer’s schema contradicts the trust rule. **Proposal:** define provider-owned legacy signatures and an explicit strip/validate/adapt order. The peer must never supply the executable validation schema. + +8. **Major — §6: dependency recovery has no retained wake-up mechanism.** A becomes broken because B is broken, releases its references, and receives no automatic retries. B later recovers through another reference’s `retry()`, but A has nothing observing that recovery. “Recovery … is immediate” therefore does not follow. **Proposal:** retain dependency-recovery subscriptions independently of attempt-owned references, and distinguish dependency failure from an intrinsically broken factory. + +9. **Major — §§4, 25: reconciliation alone leaves a post-revocation dispatch race.** A guest-origin Durable task is paused for approval. Its grant is revoked, but approval resolves before the shared store’s reconciliation routine runs; the next tool effect starts under the host principal. **Proposal:** keep reconciliation for cleanup, but require current-grant checks when accepting answers and dispatching new effects. This does not require a revocation hook. + +10. **Major — §9: transaction results can escape as revoked drafts.** The advertised consistent-read pattern permits `return (await tx.draft(doc)).items`. That returns a draft-backed object, which throws after commit; wrapping it in another object does not help. **Proposal:** specify safe immutable result extraction, or explicitly prohibit escaping drafts and document how consistent reads return nested data without copying the entire document. + +11. **Minor — §§12, 30 and cost-model target: structural sharing is not O(change) for wide containers.** Updating one item in a large flat array still copies its containing array; `mapShared` still maps and compares every element. Ten viewers multiply that work despite all proposed “no walks/no deep copies” counters passing. **Proposal:** document the container-width cost and benchmark long arrays, allocations and element visits; use bounded windows or partitioned documents where needed. + +12. **Minor — §31 contradicts §§17, 24.** The rejected list still includes “dynamic registries” and “late-bound registry entries,” while runtime-changing contributions and execute-time sandbox lookup are required behavior. **Proposal:** remove or narrowly define those rejected terms so implementation does not accidentally forbid the first proof’s required handoff. diff --git a/apps/p2/docs/p3-reviews/v2-review2-fable.md b/apps/p2/docs/p3-reviews/v2-review2-fable.md new file mode 100644 index 000000000..acbfcb78c --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review2-fable.md @@ -0,0 +1,44 @@ +# Chord v2 review round 2 (design): Fable + +Reviewed `packages/chord/docs/design.md` at `92ecf063b` against the decision log and `v2-cost-model.md`. +Proposals only; the owner decides. Kept verbatim from the reviewer's answer. + +Reviewed `design.md` at `92ecf063b` against the current decision log and `v2-cost-model.md`. Findings are numbered; "§" refers to design sections. + +1. **blocker, §22 Wire / §30 "relay: header only, body forwarded opaquely" vs. §22 "one encoding per frame and link".** In p3 the only relay hop is IPC (structured clone) → WebSocket (binary envelope). A structured-clone body is not wire bytes, so the server must encode it; and a worker commit frame carries batches for several destinations (N client links plus the server's own holds), so the relay must split the body per link to keep "one state frame per commit per link". Neither the encoding point nor the split is defined, and "no decode, no walk, no re-encode" is impossible as written. Proposal: the origin encodes each hold's batch as its own byte segment (portable envelope, compressed above the threshold at the origin), IPC carries the segments as bytes, the packet header lists `(holdId → segment offsets)`; the relay slices and concatenates segments per destination link without parsing them, and only a local consumer decodes. State that compression is decided at the origin, not per link. + +2. **blocker, §27 `accepts` value adapters + §5 "never with a schema a peer sent".** During a rollout a guest tab on R−1 calls `add(argsV1)`. The provider owns only the V2 schema; the V1 schema exists only in the peer's description, which must not be executed. So `methods: { add: (argsV1) => argsV2 }` receives unvalidated, unstripped guest input, and the mapped output is code output (not re-validated). Every rollout window opens an unvalidated path for guests. Proposal: `accepts[v]` carries the provider's own copy of the old member schemas (`{ args: OldArgs, state: …, methods: … }`); the provider validates old args against that copy before mapping, then validates the mapped args against the current schema (both compiled once). + +3. **major, §9 "only `tx.draft`" vs. §9 "a transaction may just read several documents consistently and return a value" (and §9 "values read in a transaction are shared").** Drafts are Delta overlays that throw after `prepare()` ("draft handles are unusable from here on"). `read: ctx => self.tx(ctx, async tx => (await tx.draft(doc)).entries)` returns a dead proxy: a same-host caller gets throws, a remote caller's result is serialized after the tx closed and the encoder throws. The first proof hits this immediately (the approval store reading a pending question back). Only primitives can be returned today. Proposal: `tx.read(handle, key?)` returning the committed immutable revision (no overlay, no copy; the draft's current revision if drafted in this tx via Delta's materialize-on-demand), and a rule: never return draft subtrees. + +4. **major, §9 `tx.draft` over `initial()` for absent documents.** Inside a transaction there is no way to tell "absent" from "equals `initial()`"; `remove` in the playground (`if (!(await tx.get(docs, id))) return false`) cannot be written, nor idempotent "create if missing with tags" vs. "update". Proposal: `tx.draft` returns `{ draft, exists }`-style information or `tx.exists(handle, key)` (metadata lookup, O(1) once loaded). + +5. **major, §13 close may drain vs. §21 restart order (close, factory, reopen) vs. D2 "a pinned call starts on the new sandbox".** Code mode reload: the old sandbox's `close` drains running scripts (minutes); §21 restarts by closing first, so the new sandbox and the tool contribution (§17: "in the table from live until it … restarts") do not exist until the drain ends; a Durable task released from an approval hook finds no tool entry and no sandbox. Also undefined: how the old entry's `execute()` finds "the current sandbox" (the old attempt's `self.open` handle never changes; a module variable is forbidden), and what it does while the registry is momentarily empty. Proposal: on reload Chord opens the new attempt first and lets the old `close` drain in the background bounded by `closeTimeout`; the sandbox is a registry value (`codemode.sandbox`) the object contributes, swapped atomically at the new attempt's `live`; `host.registry(R).when(pred, signal)` with the park deadline for readers that must wait. + +6. **major, §24 start barrier "open without scheduling, wait for the required contributions".** The harness waits inside its opener for `HarnessModels`; if the models plugin is missing or its native fails (`broken`), the harness never goes live, the start barrier never completes, holds park 60 s and fail `unavailable` with no reason, and keep-alive (no viewers, no tasks) stops the worker 5 minutes later; the operator sees nothing. Proposal: the harness opens unconditionally and publishes `{ kind: "waiting", missing: [...] }` state; a routine on the registries resumes scheduling when the required contributions appear; the start barrier counts it as live. + +7. **major, §5 validation by trust: stripping and normalization for host principals are unspecified.** "Not validated in production" for server/worker code also skips stripping ("a handler receives exactly its declared shape") and, if read literally, input normalization. Scenarios: (a) a worker routine calls `catalog.update({ title, archived })` where the handler does `Object.assign(draft, args)`; the trusted path mass-assigns while the guest path cannot, so behaviour differs by caller; (b) an unnormalized input (defaults omitted) from a worker yields a second object identity for the same logical input. Proposal: always normalize inputs (identity) and strip args to the provider schema (O(args), args are small by design); skip only the schema check for host principals. + +8. **major, §10 "checkpoints optional, no default; replay only when loading" + memory storage "stores ops".** A status/usage singleton written every second for a day accumulates ~86k deltas; the next worker start replays them all (and the SQLite file and the memory storage grow without bound; keyed docs evicted after 10 min replay their whole tail on every re-access). K9 moved the quadratic cost from commit to load, it did not remove it. Proposal: a store-level default (`deltasSinceBase > 64` or delta bytes > 2× base) that a definition can override, applied in the store, not the storage. + +9. **major, §3 flow control "the window applies to every frame, first snapshots included" + "a relay whose window overflows releases the hold and re-holds; the provider sends a fresh snapshot".** A 500 KB state and a 256 KB window: the re-hold snapshot alone exceeds the window, so either it can never be sent (deadlock) or it is sent and immediately overflows again (re-hold storm, one full encode per cycle). Proposal: overflow is decided on queued deltas only; a snapshot is always admitted (it consumes credits); after an overflow the hold is marked "snapshot pending" and re-held only once credits cover the last snapshot's size, with a minimum interval. + +10. **major, §15 D4 "a running job counts as a reference; a run ignoring the signal keeps its slot" vs. §4 revocation and §21 native restart.** A revoked guest's view object with a job that ignores the signal stays alive indefinitely after `revoke`; its job can keep calling out. The job's `ctx.principal` is undefined in the design. Proposal: jobs run as the host principal with attribution as data (state it); revocation and native restart abort the object's jobs and end the object when they settle or at `closeTimeout`, reported as a leak. + +11. **major, §9 versions "the migrated base is persisted" + D9 "values are not walked".** `migrate` output and `initial()` are never checked in production. A buggy `migrate` (the one time a whole document is rewritten) stamps corrupt data with the new fingerprint, permanently. Proposal: validate `migrate` output and `initial()` against the schema always (once per document per bump, off the hot path); keep D9 for everything else. + +12. **major, §25 "ignores reports from an older worker incarnation".** The server object receives the report as a method call from the worker principal; the principal carries no incarnation, and the host API exposes no way for server app code to learn the current child incarnation per address. Proposal: the worker sends `host.incarnation` as data, and `route`/the host exposes `host.children` (address → incarnation, live) or a `resolved` event so the receiver can compare. + +13. **minor, §5 description once per (link, fingerprint).** As written every first hold of a type per link costs a round trip (hold → describe request → description → bind). The consumer knows whether it already sent that fingerprint on this link. Proposal: the consumer includes the description with the first hold of a fingerprint per link; the provider still asks if its cache misses (reconnect races). + +14. **minor, §17 runtime composition.** Undefined: which objects may contribute. A `lifetime: "referenced"` object's contribution flickers with its viewers; a per-principal `remote` object would add one entry per guest. Proposal: contributions only from facets and `lifetime: "host"` objects (type-checked), or state the flicker explicitly. + +15. **minor, §22 "no frame is walked" / §30 receive "header check".** Overbroad: the frame grammar (ids, `kind`, `args.kind`, batch shapes) still needs a structural check on untrusted links, and `state`/`view`/`result` frames must be refused by kind on accepted (client) links. Define: grammar check O(frame structure) with `Unknown` leaves (TypeBox Check as today), no `isValue` walk, kind allow-list per link kind; bytes recognized by `instanceof Uint8Array` (Node `Buffer` arrives as a subclass over IPC). + +16. **minor, §29 telemetry across relays.** Relays "add a `chord.relay` span" and providers extract carriers, but relays do not decode bodies (1). Proposal: the carrier lives in the packet header next to the route; cap carrier size for frames from accepted links. + +17. **minor, §6 dependencies after `live` (K3).** The design still describes dependency waiting only before `live`; what happens when a declared reference fails or is released after `live` (dependent stays live with stale values, or goes `loading`/`failed`) is not written. Proposal: a declared reference that leaves `live` marks the dependent `loading` (calls park) and fails it `transient` after `readyTimeout`, matching activation. + +18. **minor, §3/§9 immutability by convention.** Concrete failure: a routine does `sessions[id].archived = true` on a value it received from `catalog.sessions`; the tracker's authority is mutated without ops, the next delta is computed against an assumed-unchanged revision, and every remote replica diverges permanently with no error (dev catches it only if that path runs under `dev`). Proposal: production freezes only the root object of every committed revision and every published value (`Object.freeze(root)`, O(1), catches `value.x =` and `Object.assign(value, …)`), deep freeze stays dev-only. + +19. **minor, drift.** §5/§22 frame list lacks the description-request frame; the log's cost-guard bench still names JSONL though cut; §9 "values read in a transaction are shared" predates `tx.get` removal; the keep-alive counters ("Chord's counters") have no named API in §3/§19. diff --git a/apps/p2/docs/p3-reviews/v2-review2-summary.md b/apps/p2/docs/p3-reviews/v2-review2-summary.md new file mode 100644 index 000000000..89d969178 --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review2-summary.md @@ -0,0 +1,55 @@ +# Chord v2 review round 2 (design): decisions for the owner + +Sources: `v2-review2-fable.md` (F), `v2-review2-astra.md` (A), gaps from the walkthrough update (W). Every item is a +question for the owner with a **proposal**; nothing here is decided. + +## A. Wire, relays, untrusted input + +| # | Sev. | Problem | Proposal | +| --- | --- | --- | --- | +| R1 | blocker | "Relays forward opaquely" cannot work: the server receives structured clone over IPC and must send binary on WebSockets, and one worker commit carries batches for several destination links; forwarding one body to all leaks another client's state (F1, A1) | The origin encodes each hold's batch as its own byte segment (compression decided at the origin); the packet header lists hold id → segment; the relay routes each segment only to the link that holds it, concatenating per destination without parsing; each destination assembles its own atomic commit frame | +| R2 | major | Compressed client frames can expand enormously before anything checks them, and the relay carries them on to IPC, which has no limit (A3) | Bound compressed size, decoded size and framing metadata separately; enforce the expansion limit while decoding; keep the limits across relays | +| R3 | major | Contract-description cache per (link, fingerprint) can be poisoned: the worker's parent link carries many principals; a guest sends a wrong description under a real fingerprint and later owner bindings use it (A2); every first hold costs a round trip (F13) | Recompute the fingerprint from the description before caching; a mismatch fails only that binding; the consumer sends the description with the first hold of a fingerprint per link, the provider still asks on a cache miss | +| R4 | minor | "No frame is walked" is too broad: untrusted links still need a grammar check of the frame structure and a kind allow-list (a client must not send `state`, `view`, `result`) (F15) | A structural frame check with opaque leaves (no value walk) and a kind allow-list per link kind; bytes by `instanceof Uint8Array` (Node `Buffer` is a subclass) | +| R5 | minor | Relay spans need the trace carrier, but relays do not decode bodies (F16) | The carrier lives in the packet header next to the route, size-capped on accepted links | + +## B. Validation and trust + +| # | Sev. | Problem | Proposal | +| --- | --- | --- | --- | +| R6 | blocker | During a rollout an old guest tab calls `add(argsV1)`; the provider only owns the V2 schema, the V1 schema exists only in the peer's description (which must not be executed), so the adapter receives unvalidated guest input (F2, A7) | `accepts[v]` carries the provider's own copy of the old member schemas; order: strip and validate against the provider's old schema, adapt, then (in dev) validate the result against the current schema; both compiled once | +| R7 | major | "Not validated for host principals" also skips stripping and input normalization: a worker can mass-assign extra fields a guest cannot; an unnormalized input from a worker makes a second object identity (F7, W4) | Always normalize inputs and strip arguments to the provider schema (cheap: arguments are small); skip only the schema check for host principals | +| R8 | major | `migrate` output and `initial()` are never checked; a buggy `migrate` writes corrupt data under the new fingerprint, permanently (F11) | Validate `migrate` output and `initial()` always (once per document per bump, off the hot path) | +| R9 | minor | Immutability by convention fails silently: `sessions[id].archived = true` on a received value mutates the authority without ops; replicas diverge for good (F18) | In production freeze only the root of each committed revision and published value (`Object.freeze(root)`, O(1)); deep freeze stays dev-only | +| R10 | major | Revocation race: a guest-started Durable task waits for approval; the grant is revoked; the approval resolves before the reconciling routine runs; the next effect runs as the host (A9) | Keep reconciliation for cleanup, and check the current grant when an answer is accepted and when a new effect is dispatched on the guest's behalf | +| R11 | major | A job of a revoked guest's view keeps its object alive (D4) and may keep calling out; a job's `ctx.principal` is undefined (F10) | Jobs run as the host principal with attribution as data; revocation and native restart abort the object's jobs, and the object ends when they settle or at `closeTimeout` (reported as a leak) | + +## C. Transactions and documents + +| # | Sev. | Problem | Proposal | +| --- | --- | --- | --- | +| R12 | major | With only `tx.draft`, a transaction cannot return read data: draft proxies throw after the commit, so `return (await tx.draft(doc)).items` fails for same-host and remote callers alike; the first proof needs it (F3, A10) | (a) `tx.read(handle, key?)` returning the committed immutable revision (or the current draft revision materialized), plus the rule "never return a draft subtree"; or (b) reads only outside transactions (`handle.value`, `handle.get`), transactions only write and return primitives | +| R13 | major | Inside a transaction "absent" and "equals `initial()`" look the same; "remove if present" or "create with tags if missing" cannot be written (F4) | `tx.draft` also tells whether the document existed (`{ draft, exists }`), or `tx.exists(handle, key?)` (metadata, O(1)) | +| R14 | major | Without a default checkpoint the cost only moves to load time: a status document written every second accumulates 86,000 deltas a day, replayed on the next start and on every reload of an evicted keyed document (F8) | A store-level default (e.g. a new base after 64 deltas or when deltas exceed twice the base), overridable per definition; Durable leaves it to each document | +| R15 | major | Old calls vs. migration: an old call awaits something, a reload migrates its document, the call resumes with its "still valid" old handle; §9 says it throws, §21 says old calls keep working (A6) | Keep §9 (old code reading a migrated document throws, Durable's rule) and narrow §21: old handles stay valid until their calls settle, except for documents whose version rose | + +## D. Lifecycle, reload, the first proof + +| # | Sev. | Problem | Proposal | +| --- | --- | --- | --- | +| R16 | major | Code mode reload: the restart closes the old object first; its `close` drains running scripts for minutes; until then there is no new sandbox and no tool entry, so a call released from an approval finds neither; how the old entry finds "the current sandbox" is undefined (F5, A5) | On a native restart Chord opens the new attempt first and lets the old `close` drain in the background (bounded by `closeTimeout`); the sandbox is a registry contribution swapped atomically when the new attempt goes live; readers that must wait use a cancellable "wait until present" with the park deadline | +| R17 | major | The harness start barrier is circular (the barrier waits for host-lifetime objects including the harness, the harness waits for the barrier) or premature; if the models plugin is missing or broken the harness never goes live, holds fail after 60 s without a reason, the worker stops idle (F6, A4) | The harness goes live while paused and publishes `{ kind: "waiting", missing: [...] }`; a routine on the registries resumes scheduling once the required contributions are present | +| R18 | major | Dependency recovery: A fails `broken` because B is broken and keeps no references, so nothing wakes A when B recovers; after `live`, what a dependency's failure does to the dependent is unwritten (A8, F17) | A dependent of a broken dependency is not itself broken: it waits on the dependency (a retained recovery subscription) and recovers with it; after `live`, a declared reference leaving `live` makes the dependent `loading` (calls park) and fails it `transient` after `readyTimeout` | +| R19 | major | Flow control: a 500 KB snapshot does not fit a 256 KB window; after an overflow the re-hold snapshot overflows again (re-hold storm) (F9) | Snapshots are always admitted (consuming credits); overflow is decided on queued deltas only; after an overflow re-hold only once credits cover the last snapshot's size, with a minimum interval | +| R20 | major | Pub/sub fence: the server object cannot learn the reporting worker's incarnation (principals carry none) (F12) | The worker sends `host.incarnation` as data, and the host exposes its children's current incarnations (address → incarnation) | +| R21 | minor | Which objects may contribute to registries: a referenced object's contribution flickers with its viewers; per-principal objects would add one entry per guest (F14) | Contributions only from facets and `lifetime: "host"` objects (type-checked), or document the flicker | +| R22 | minor | A host-lifetime object without input (e.g. `NoteInbox`) can no longer `self.get` a session-scoped object in its factory, because start data is read only after activation (W2) | Document the pattern: such objects `ctx.get` session-scoped objects in routines; or allow `self.get` with an input computed after activation | +| R23 | minor | `constant(init)` is used in §18 but not defined (W1) | A small readable helper `constant(value)`, or members that accept a plain value as constant state | + +## E. Costs and documentation + +| # | Sev. | Problem | Proposal | +| --- | --- | --- | --- | +| R24 | minor | Structural sharing is not O(change) for wide containers: changing one item of a large flat array copies that array; `mapShared` still maps and compares every element (A11) | Document the container-width cost; benchmark long arrays (allocations, element visits); partitioned documents or bounded windows where needed | +| R25 | minor | §31 still rejects "dynamic registries" and "late-bound registry entries", while contributions now change at runtime and code mode looks up at execute time (A12) | Narrow the wording: rejected are registries created at runtime and entries resolved through mutable indirection, not contributions changing with their contributors | +| R26 | minor | Drift: no description-request frame in the frame list; the log's benchmark still names JSONL; §9 "values read in a transaction are shared"; keep-alive counters have no API name (F19) | Fix the text | diff --git a/apps/p2/docs/p3-reviews/v2-review3-astra.md b/apps/p2/docs/p3-reviews/v2-review3-astra.md new file mode 100644 index 000000000..2e3432d8d --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review3-astra.md @@ -0,0 +1,431 @@ +# Chord v2: final design review — Astra + +Reviewed `origin/private` at `138268fa3`. +Read the design, decision log and round-2 summary in full; +incorporated the subsequent `host.activity` change. +This reviews the target, not the frozen implementation. + +**Overall: thumbs down for final sign-off.** +The remaining problems are specific protocol, ownership and lifecycle gaps, +not a request to restore the discarded validation or copying layers. +R9, R10, R12 and R13 are treated as owner decisions, not reopened here. +All changes below are proposals. + +## A. Verdict per area + +- **Hosts and links — thumbs down.** Oversized-snapshot recovery can wait forever (D3). +- **Principals and revocation — thumbs up.** No new blocker beyond the explicitly accepted revocation window. +- **Object types and compatibility — thumbs down.** Old fingerprints cover more than `accepts` supplies (D5); validation rules also disagree (B1). +- **Objects and lifecycle — thumbs down.** Retained failed dependencies need an explicit cycle-rejection rule (D4). +- **References — thumbs down.** Late replies after cancellation/release lack a safe retirement protocol (D1). +- **Ctx — thumbs up.** Keep one owned call scope; fold `ctx.incarnation` from §25 into the §8 API definition. +- **Documents and store — thumbs down.** Absent singleton reads and facet access permissions contradict themselves (B3–B4). +- **Derives — thumbs up.** No new blocker; do not add a second projection engine. +- **Natives — thumbs up.** One sequential open/close lifecycle suffices; the sandbox exception needs consistent wording (B6). +- **Routines — thumbs up.** Implement the four forms over one runner, not four lifecycle engines (C4). +- **Jobs — thumbs down.** Completed-record, dedupe-key and waiter retention has no contract (D7). +- **Facets — thumbs up.** Reuse the object machinery; consider removing the cross-owner transaction exception (C2). +- **Registries, including `self.writable` — thumbs down.** The authoritative definition/composer across reloads is unspecified (D6). +- **Start data — thumbs up.** No further primitive is needed beyond an ordinary object and constant readable. +- **Reload and rollout — thumbs down.** The harness is simultaneously inside and outside the barrier (B2); registry-definition replacement is also unresolved (D6). +- **Wire — thumbs down.** D1–D3 prevent signing off the protocol. +- **Telemetry — thumbs up.** Reuse the existing adapter and span APIs; do not add another instrumentation abstraction. +- **Cost model — thumbs down.** Its counting example asserts the superseded encoding unit (B9). + +## B. Contradictions + +Quotes are from `design.md` unless identified as log entries. +Line numbers refer to the reviewed revision. + +### B1. Validation still has three incompatible descriptions + +§3, lines 237–238: +> Values are validated only where they enter from an untrusted caller + +§5's table, line 288: +> validated once at the provider when the caller is not a host principal + +§5, lines 312–314: +> Method arguments are always stripped to the provider's schema and validated with it, whoever the caller is + +§27, line 1343: +> v1 arguments are validated against the provider's v1 schema (untrusted callers, as in section 5) + +Log R7 says: +> Method arguments are always stripped to the provider's schema and validated with it (compiled once), whoever the caller is + +Apply R7 consistently. Also explicitly resolve whether the table's local +“neither validated nor copied” remains an exception to the unqualified rule. +Do not let the implementation infer the trust boundary from whichever paragraph +it encounters first. + +### B2. Harness activation both is and is not the barrier + +§24, line 1235: +> its activation is the start barrier + +§21, lines 1088–1089: +> the Durable harness starts after the barrier, it is not part of it + +Log R17 chooses the latter. Remove the former and specify how app startup +expresses the ordering without a Harness-specific branch inside Chord (C7). + +### B3. An absent singleton has two different values + +§9, lines 548–549: +> `handle.value` / `handle.get` give `undefined` for an absent one + +§9, lines 570–571: +> A singleton that does not exist yet reads `initial()` through its handle; retiring resets it to `initial()`. + +These cannot both describe `handle.value`. Choose one singleton rule; +keep the keyed-document absence rule separate. This is not a request for +transactional existence checks. + +### B4. The facet-document permission is reversed in one paragraph + +§9, line 571: +> Handles may be used only in transactions of their own object (or of the facet that serves it). + +§9, lines 580–581: +> A served object's transaction may span its own documents and its facet's documents +> the facet never reaches into a served object's documents. + +The first sentence permits the forbidden direction. A served object may use +its facet's handles, not the reverse. + +### B5. The opening document example violates required checkpointing + +§9's complete `Settings` declaration is: + +```ts +const Settings = defineState("settings", { + schema: SettingsSchema, + version: 1, + initial: () => ({ theme: "dark" }), + persist: true, // default: in memory +}); +``` + +§10, line 647: +> `checkpointWhen` (required on every definition, no default, R14; as in Durable) + +Add an explicit policy to the example. Otherwise the first copied example +fails the chosen API. + +### B6. Code mode's close still describes the superseded drain + +§24, lines 1257–1258: +> the old sandbox is sealed and drained in `close` (running scripts finish or hit their deadline, then it ends) + +§13, lines 766–767: +> `close` detaches the old resource, returns at once, and disposes of it when its work has finished or after `closeTimeout` (30 s) + +Log R16 is explicit: +> the default `closeTimeout` (30 s, no override) + +Use the R16 rule consistently: detached background disposal, with a hard +30-second bound for code mode, not an arbitrary script deadline or a blocking close. + +### B7. “Nothing is copied” conflicts with stripping shared arguments + +§5, lines 312–317: +> Method arguments are always stripped to the provider's schema +> Nothing is copied in process: same-host calls pass values by reference + +§3, line 239: +> nobody mutates a value it got + +Pass a shared `{ allowed, private }` object to a method accepting only +`{ allowed }`. Stripping cannot both preserve that object and pass it unchanged. +Specify **no unconditional defensive clone**, with copy-on-change normalization. +Do not implement stripping by deleting fields from shared arguments. + +### B8. Accepted fingerprints need more than member schemas + +§5, line 306: +> The fingerprint covers input, variants and members, not placement. + +§27, lines 1340–1342: +> `accepts[v]` carries the provider's own copy of version v's member schemas plus value mappings +> The provider computes v1's fingerprint from those schemas + +The old input schema is missing. This becomes a functional failure, not just +wording, when the input changes (D5). + +### B9. Cost guards count links instead of encoded parts + +§30's table, line 1440: +> one encoding per hold part at its origin (that hold's Delta encoder) + +§30's test example, line 1450: +> one encode per link + +R1 explicitly says two holds encode separately. Ten client holds multiplexed +over one worker IPC link require ten part encodes, not one. Count part encodes +at the origin, message assembly separately, and zero payload encodes at relays. + +### B10. The tagged-union rule is not followed by its examples + +§1, line 33: +> States, results and outcomes are discriminated by `kind` + +§15: +> Outcome: `{ ok: true, result } | { ok: false, error: "threw" | "aborted" | "lost" }` + +§20: +> `state { status: "running" | "done" | "failed" | "cancelled", step?: { id, kind, … }, notices }` + +Use the chosen discriminant consistently before these become public types. +The flow example also permits a completed flow with an active step. + +### B11. The rejected list still rejects the new cold-path checks + +§31: +> validating state and documents by walking values in production + +§9, lines 594–596: +> the result of `migrate` and of `initial()` is validated against the new schema once, when it is produced + +Narrow the rejected item to repeated state/commit validation; retain R8. + +## C. Primitives to remove or merge + +These are simplification proposals, not implementation instructions. +Do not estimate a 6–7k-line core while silently counting another runtime +inside optional helpers as free. + +### C1. Move the job engine out of the mandatory kernel + +**What:** job definitions, persisted records, dedupe, concurrency, progress, +version adapters, current-code finish, retries and owner pinning. + +**Replacement:** an optional job component built on the same document store +and owned scopes. Keep the transaction integration there. The first proof +already uses Durable tasks and need not require a second task engine. + +**What breaks:** core-only users no longer get `self.job`/`tx.start`. +Moving jobs directly to Durable is **not** a transparent substitute: +Chord intent and Durable submission cannot commit atomically. A full removal +would require moving the whole workflow's intent/outcome into Durable. + +This is the largest credible scope cut; it requires an explicit owner choice. + +### C2. Remove facet-owned mutable documents and the transaction exception + +**What:** transactions spanning a served object and its facet, with two lines +and a special ownership permission. + +**Replacement:** facets declare types and contributions; mutable shared data +belongs to an ordinary local object. Put data needing one atomic invariant +under that one owner. + +**What breaks:** existing designs atomically updating instance documents and +facet documents must move those writes into the shared owner. Keeping the +current split while replacing the second lock with a method call is incorrect. + +This trades some ownership flexibility for one transaction rule everywhere. + +### C3. Merge writable, constant and derived subscription machinery + +**What:** `self.writable`, `constant`, derives and registry-composed readables +should not each introduce a subscriber list and disposal implementation. + +**Replacement:** reuse one host-local cell/subscription substrate. +`constant` has no writer; `self.writable` is an attempt-owned writer; +derives and registry composition are managed computations. +Keep `replicatedState`'s ordered-op/snapshot interface for replicated data. + +**What breaks:** nothing if the public contracts remain. +Do **not** make arbitrary native-carrying cells exposable as wire state. +Old-generation setters must not update new-generation contributions; +ending an attempt must detach its subscriptions. + +### C4. Merge the four routine forms into one owned runner + +**What:** `on`, `every`, `loop`, `custom`. + +**Replacement:** one named runner owning cancellation, call scope, cleanup, +error reporting and reload handover. The four forms are small setup helpers. + +**What breaks:** nothing if `every` retains its next-due timestamp by name +and `on` keeps gapless attachment. Four separate state machines would make +those guarantees harder, not stronger. + +### C5. Merge acquisition APIs into one lease implementation + +**What:** `host.hold`, awaited `host.get`, `self.get`, `ctx.get`. + +**Replacement:** one acquisition returning a lease and one readiness waiter. +Factory/call/host APIs choose the lease owner; awaiting is convenience. +Use the same owned-scope cleanup for call cancellation and reference release. + +**What breaks:** nothing if failed/aborted awaits release their lease and +parked calls retain their reference-specific deadline. Do not merge the +provider object's lifetime with a consumer lease's lifetime. + +### C6. Move registry-version adaptation into composition + +**What:** the kernel's registry `accepts` engine and newer-version load refusal. + +**Replacement:** compose receives versioned contributions. A pure helper +adapts supported versions; the owner omits and reports unsupported ones, +like other runtime composition conflicts. + +**What breaks:** unsupported newer contributions no longer reject a load; +they become an owner notice and are excluded. If that load-time guarantee is +required, retain the metadata check but still share one composition path. + +Do not replace document or remote-contract version checks this way: +those protect different boundaries. + +### C7. Remove the Harness-specific startup exception from Chord + +**What:** “all host-lifetime objects except the harness.” + +**Replacement:** register Harness as an ordinary referenced object; app startup +holds it after `await host.load(...)` and releases it at shutdown. +Pre-barrier contributors look up Harness lazily when executing work. + +**What breaks:** pre-barrier factories/openers cannot eagerly depend on Harness. +Such dependencies already defeat R17: they wait for a post-barrier object, +time out, and falsely appear as failed plugins before Harness starts. + +This keeps the owner's post-barrier startup decision without teaching the +portable kernel a p3 service name or adding another lifetime mode. + +**Do not add:** a job-object lifetime, a second flow protocol, a projection +algebra, a native-replacement engine, or a second retry scheduler. +Use ordinary addressed views, owner scopes and the existing readable path. + +## D. Remaining blockers and majors + +### D1. Blocker — ordinary cancellation can close a healthy link + +**Sections:** 7, 22; R4. + +Trace: +1. Client cancels call C and retires its pending entry. +2. Provider's result for C was already sent in the opposite direction. +3. Result arrives: C is now an unknown id. +4. §22 closes the link, failing unrelated calls and holds. + +The reverse race is equally valid: the provider completes and retires C +before the client's cancel arrives. Release versus an already-sent state +part has the same problem. Ordered delivery within each direction does not +order the two directions against each other. + +**Proposal:** specify retiring states and terminal acknowledgements, using +`result`/`gone` where possible. Distinguish legitimately retired ids from +never-issued ids. Bound the retirement bookkeeping and test its reclamation; +keeping every canceled entry forever is not an alternative. + +### D2. Major — the decompression budget can disappear at the relay + +**Sections:** 3, 22; R1–R2. + +Trace: a guest sends a small compressed call that expands past 64 MiB. +The server checks `maxPayload` and forwards the opaque part over IPC. +The worker performs the only decode. Applying the stated “none on IPC” +policy there bypasses the client-WebSocket decoded limit. + +There is a second incompatible promise: `ws.maxPayload` rejects the whole +WebSocket message with close code 1009. It cannot report only that call as +`too_large` while preserving the link. The installed ws receiver confirms +this behavior; it is not a hypothetical application choice. + +**Proposal:** carry a trusted ingress decode budget with forwarded work, +independent of the final physical link. Stamp it at admission, not from a +client-controlled field. Define per-part and aggregate decoded limits. +Separate sender/application `too_large` errors from transport-limit breaches: +the latter close the connection. No payload walk is required. + +### D3. Major — R19's re-hold condition is impossible for an oversized snapshot + +**Section:** 3, Flow control; R19. + +Trace with a 256 KiB window and a 500 KiB snapshot: +1. The snapshot is admitted; credits become negative. +2. Changes overflow while it is in flight; the hold is released. +3. Its acknowledgement restores the window to 256 KiB. +4. Re-hold waits for credits covering 500 KiB. That can never happen. + +The initial re-hold storm has been replaced by a permanent stall. + +**Proposal:** permit one oversized snapshot when outstanding snapshot debt +has drained, rather than requiring free credits to exceed its size. +Keep the minimum re-hold interval. Specify the accounting equation and test +this exact case plus a growing replacement snapshot. + +### D4. Major — retained failed dependencies can form an immortal component + +**Sections:** 6, 28; R18. + +Trace: +1. A's factory gets B; B's factory gets A. +2. Both time out and become transient failures. +3. Both retain their dependency references for recovery. +4. The external caller releases A. +5. Neither object loses its last reference; retries and retained generations + can survive indefinitely. + +§28 requests cycle tests but does not define the expected outcome. + +**Proposal:** reject declared dependency cycles before activation waits, +release the cycle's acquisition leases, and report a broken declaration. +Do not introduce tracing garbage collection for object-reference cycles. +Test the cycle introduced by a reload as well as first activation. + +### D5. Major — accepted versions cannot reconstruct old input identity + +**Sections:** 3, 5, 27; R6–R7. + +Trace: V1 input is `{ session }`; V2 changes it to `{ sessionId }` and +advertises an `accepts[1]` entry containing V1 member schemas. +The V1 fingerprint includes its old input schema, which that entry lacks. +Normalizing the V1 hold with the current schema also drops `session` before +placement and object identity can be resolved. + +**Proposal:** retain the complete provider-owned old contract, not only its +members. For the minimal first implementation, explicitly prohibit input +shape/identity changes through `accepts`; require a new object type for them. +If they must be supported, define validated input adaptation before canonical +current identity and placement. Do not improvise it from peer data. + +### D6. Major — registry definition ownership is missing during reload + +**Sections:** 17, 21, 27. + +Trace: plugin A owns registry R and reloads its composition to V2. +Unchanged plugin B still has an imported V1 token, including its old +`compose`, and calls `host.registry(R1)` or contributes through it. +The design says how contribution values are adapted, but never says which +registered definition owns R, how that definition is replaced, or how an +old reader token is prevented from reinstalling the old composer. + +First-token-wins leaves stale behavior; last-token-wins makes behavior depend +on activation timing. Keeping one table per token splits one extension point. + +**Proposal:** give the defining facet one authoritative declaration for R. +Other tokens identify a registry but cannot install its composer. Reload +replaces that declaration atomically; specify reader-version compatibility +and disposal of superseded subscriptions. Reuse the facet declaration table, +not a second loader or lifecycle engine. + +### D7. Major — job dedupe and result retention have no end condition + +**Section:** 15. + +Trace: a host-lifetime owner completes 100,000 uniquely keyed jobs. +If acknowledged records and key-to-run entries remain, the live process +retains an unbounded history. If they are dropped immediately, retrying a +completed request with the same key starts a second effect instead of +returning the existing run. Outstanding `outcome(ctx)` handles add another +retention owner. No lifetime or collection rule is specified. + +**Proposal:** define the dedupe horizon, record retirement and result-handle +ownership together. State explicitly whether dedupe covers only unsettled +runs or a retained completed history. Release all waiter and scope references +when settled; persisted history must not imply a permanent in-memory cache. +Test completed runs, repeated keys, blocked finishes and removed definitions. +If the full job component is deferred (C1), defer this complexity with it. diff --git a/apps/p2/docs/p3-reviews/v2-review3-fable.md b/apps/p2/docs/p3-reviews/v2-review3-fable.md new file mode 100644 index 000000000..140f5dc07 --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review3-fable.md @@ -0,0 +1,274 @@ +# Chord v2 design review, round 3 (fable) + +Read fresh from `origin/private` at `1fbadcf1e`: +`packages/chord/docs/design.md`, `packages/chord/docs/decisions.md`, +`apps/p2/docs/p3-reviews/v2-review2-summary.md`. +"§n" = design section. "R n" = round-2 decision. Quotes are verbatim. + +## A. Verdict per area + +| Area | Verdict | One line | +| --- | --- | --- | +| hosts and links | up | small API; IPC flow control text is stale (B6) | +| principals and revocation | up | R10/R11 close the holes; relayed `ctx.incarnation` undefined (B9) | +| object types and compatibility | down | `local`/`remote` split and the whole `accepts` path are the biggest removable surface (C1, C2) | +| objects and lifecycle | up | R18 is right; `broken` for opens plus `retryOn` is a second trigger system to cut (C6) | +| references | up | `state: false` has no wire support any more (B5); cut it (C8) | +| ctx | up | `ctx.trace` duplicates `traceIn` (C11); incarnation on relays undefined | +| documents and store | up | Durable port; R12/R13 patterns work; examples violate R14 (B4) | +| derives | up | `previous` is dead weight next to `mapShared` (C9) | +| natives | up | R16's 30 s timer contradicts §13 "scripts finish or hit their deadline" (D4) | +| routines | down | four forms plus handover for what is two (C4) | +| jobs | down | job versions, `accepts`, `blocked`, `persist` duplicate documents (C3) | +| facets | merge | a facet is a host-lifetime object that may not open natives; drop the restriction and `lifetime: "host"` collapses into it (C5) | +| registries incl. `self.writable` | up | registry `accepts` removable (C2); `writable`/`constant` are one thing (C10); non-JSON writables as state members undefined (B10) | +| start data | up | — | +| reload and rollout | up | with "install before routed" the adapted-binding path never runs in p3 (C2) | +| wire | up (R1) | "unknown id closes the link" is a race, not a violation (D1); per-message size limit kills whole commits (D3) | +| telemetry | up | log's Telemetry section still says `TelemetryContext` member (B8) | +| cost model | up | — | + +Overall: up, after D1 and D2 are fixed and the C list is decided. +The design is coherent enough to build; it is not yet minimal. + +## B. Contradictions + +B1. Validation of trusted callers. §5 table: "arguments: validated once at the provider when the +caller is not a host principal". §5 bullet (R7): "Method arguments are always stripped to the +provider's schema and validated with it, whoever the caller is". §3 Values: "Values are +validated only where they enter from an untrusted caller (section 5)". §27: "v1 arguments are +validated against the provider's v1 schema (untrusted callers, as in section 5)". R7 wins; the +table, §3 and §27 are stale. + +B2. Harness and the start barrier. §21: "wait until every facet and `lifetime: "host"` object +other than the harness is live or failed … the Durable harness starts after the barrier, it is +not part of it; R17". §24: "a `Harness` object with `lifetime: "host"` (… its activation is the +start barrier)". §24 is stale. + +B3. Harness registry. §24 example passes `registry` once to `Durable.open`. §17: "Chord +subscribes to readable contributions … on every change Chord replaces that entry … and runs +`compose` again"; R25: "readers may look up the current value at use". The example must pass +`() => host.registry(HarnessExtensions).value`, like `models`. + +B4. Required checkpoints vs. examples. §10: "`checkpointWhen` (required on every definition, no +default, R14)". §9 `Settings` and §15 `ShellResult` definitions have none. + +B5. Method-only references vs. the wire. §7: "`state: false` binds for methods only". §22 kinds: +"`hold`, `release`, `call`, `cancel` … `bound` (1 part: snapshot)". No `sub`/`unsub` exist any +more, and `bound` always carries a snapshot. Either `hold` carries a `state` flag and +upgrading is a re-hold, or `state: false` goes (C8). + +B6. IPC flow control. §3: "a window of frame counts on IPC (Node serializes internally; nothing is +stringified to count)". §22 (R1): "IPC: structured clone of `{ header, parts }` … Flow control +counts the parts' sizes from the header". Parts are bytes on IPC now; the count window is stale +and unbounded in bytes. + +B7. Barrier promise vs. retrying facets. §21: "Then the registries hold everything that will be +there". §6: "Facets follow the same rules: a transient activation failure retries with backoff". +A facet in backoff counts as "failed" for the barrier and contributes later. R17's "model +unavailable, wait" covers it; the sentence is false. + +B8. Telemetry types. Log, "Telemetry": "Chord's `Context` gets `telemetry: TelemetryContext` as a +fixed member … `withTelemetry(telemetry, context)`". §29: "`Context.telemetry` is always a +`TelemetrySpan` … `withTelemetry(span, context)`". The log entry is not marked superseded. + +B9. Incarnation on relayed calls. §25 (R20): "a call arriving over a link carries the calling +host's incarnation from the link handshake". For client → server → worker the worker's handshake +is with the server. Which incarnation the worker sees is undefined; relays must stamp the +origin's (they already stamp the principal, §22). + +B10. `self.writable` as a state member. §17: "a host-local readable of any value (no JSON, no +diff, no wire, compared by identity)". §5: members are "state and methods (JSON plus bytes)". +Nothing says a writable holding a function is refused as an exposed member; it would reach the +encoder. + +B11. Singleton existence. §9 (R13): "existence is read outside the transaction (`handle.value` / +`handle.get` give `undefined` for an absent one)". §9: "A singleton that does not exist yet reads +`initial()` through its handle". Singleton existence cannot be read at all. + +B12. Log entries superseded but unmarked (the log says they are "marked inline"): "Derives and +natives step": "a `broken` dependency fails it at once. No retained references while failed" +(superseded by R18); "Registries and HTTP": "a conflict it reports during staging rolls the host +back" and "compose … throwing on a conflict … in staging" (superseded by D1); "Reload": "No +per-module digest: if a native is expensive to restart, the author splits it into its own plugin" +(superseded by D5/R16 patterns); "Object kinds": "validate and copy arguments and results also on +the same host" (superseded by the state path). + +B13. §6 wording: "On a reload the factory runs again for pure objects (new methods and derives, +same documents and natives)". Pure objects have no natives. + +## C. Primitives to remove or merge + +C1. Merge `local` into `remote` (one `object` kind). +What: two contract kinds, two factory signatures, `LocalCtx` vs `Ctx`, `LocalServe`, the +"clients serve only local" rule, two identity rules. +Why it can go: after R7 (everyone validated) and the state path (no copies) a same-host call on +a `remote` type and a `local` call run the same code path. What `local` still encodes is two +flags: not addressable from other hosts, and no principal in the identity. +Replacement: `define(name, { host, principals: false | true, … })`. `principals: false` means +identity (type, input), factory `(input, self)`, `ctx.principal` absent. A client serves only +`principals: false` types placed on itself. Variants only with `principals: true`. +What breaks: nothing in p3; the shared-store pattern is the same with one keyword fewer. +Lines: one contract family, one ref type family, one ctx type. + +C2. Remove `accepts` on object types, adapted bindings, registry `accepts`. +What: §27 provider-owned old schemas plus value mappings, §22 "adapted bindings get diffs" (a +derive and a cached value per binding), §5 "a version is one it accepts", registry `accepts`. +Why it can go: §21: "A host that starts or reconnects installs the current revision before +anything is routed to it"; rollout order server → workers → clients; a behind consumer's calls +park as `updating`. So in p3 no consumer ever binds with an old version except a plugin that was +not updated, which §21 already treats as `contract_mismatch` "final" with a notice. Registries are +host-local and a host is "completely at revision R or R−1": contributor and owner are never on +different registry versions in one revision. +Replacement: strict fingerprints only (R6 without the `accepts` half): bind iff version and +fingerprint equal; otherwise `updating` during a rollout, else `contract_mismatch`. +What breaks: a third-party plugin built against an older core contract fails to bind instead of +being adapted. Acceptable: it fails loudly, as documents without `migrate` do. +Lines: the largest single cut (old-schema fingerprints, mapping wrappers, per-binding derives and +diffs, their tests). + +C3. Shrink jobs to a helper over documents and routines. +What: `defineJob` with `version`, `accepts`, `blocked`, `persist`, `running`, `outcome`, +`concurrency`, `key`, `abort`, `finish`. +Why: the job record is a keyed document (intent in the owner's transaction: `tx.start` is +`tx.draft(records, id)` plus a dedupe key in a singleton); versions and `accepts` on outcomes are +what document `version` and `migrate` already do (a stored outcome is data in the owner's +document); `persist` is the record document's `persist`; `blocked` is a document state. +Replacement: keep `self.job(def, { run, finish, concurrency })` as a thin runner: it watches the +records document, runs `run` for pending records, commits `finish` with the outcome; `running` +is a derive over the records document plus in-memory progress. Keep "a running job counts as a +reference" (one counter). +What breaks: nothing visible; `defineJob` loses `version`/`accepts`/`persist`. +Lines: the job version/adapter/blocked machinery and its wire of `running`. + +C4. Routines: four forms to two. +What: `on`, `every`, `loop`, `custom`, plus "due time survives reloads" for `every`. +Why: `custom` is `loop` with a cleanup; `every` is `loop` with `await sleep(ms, ctx.signal)`; +"due time survives reloads" needs a Chord-kept timer per routine name. +Replacement: `routines.on(name, readable, fn)` and `routines.loop(name, fn)`; `every(ms)` is a +one-line helper on top of `loop`; a due time that must survive is a document (the owner writes +`nextAt`). Serialization per name stays, for both. +What breaks: the `every` due-time guarantee across reloads (one document line replaces it). + +C5. Merge `lifetime: "host"` objects into the facet. +What: the facet (pure, host lifetime, no natives, "cannot `self.get` a type it serves"), plus +`lifetime: "host"` objects (input-less, pinned, the only native owners that are always on, the +only contributors besides facets, R21), plus the "worker-wide native owners" and R22 patterns. +Why: both are "one host-lifetime object per plugin"; the only difference is that the facet may +not open natives. Drop that restriction: the facet opens the plugin's always-on natives (inbox +watcher, model runtime, sandbox, the harness in the core facet) and contributes. Reload: a facet +with natives restarts like a native object (its plugin's natives restart anyway). +Replacement: `defineFacet((host, self) => …)` where `self` is a full `Self` (`open`, `state`, +`routines`, `contribute`, `get` of other plugins' types). `lifetime` option removed; every +served object is referenced. Start data `SessionInfo` becomes an ordinary referenced object of +the core facet (cheap to hold; R22 goes away because a facet may `self.get` other facets' +types). +What breaks: a plugin with two independent always-on natives shares one failure domain (one +facet). If a real case needs two, split the plugin or accept it. The "facet never gets a type it +serves" rule stays. +Lines: pins, the host-lifetime branch in activation, `FacetSelf`, two documented patterns. + +C6. Remove `broken()` for opens and `retryOn`. +What: `throw broken(error)` in an opener, `self.retryOn(...readables)` with its "sources must +live independently of the attempt" rule, watches installed before opens. +Why: they exist so an open with bad configuration is not retried every 60 s. A retry every 60 s +of an invalid SSH config costs one failed connect. Config changes then recover through the +backoff (≤ 60 s) with no second trigger system. `broken` stays for code errors (factory, derive, +hydration, `migrate`), which only a reload or `ref.retry()` fixes. +What breaks: nothing; a Retry button and reload still work. + +C7. Remove provider-side `linger`. +What: `serve(…, { linger })`, the lingering state of an object, "lingering objects" in keep-alive. +Why: "keeps a just-closed conversation warm" is an app policy: a host-lifetime holder (the core +facet) keeps `host.hold(Conversation, …)` for N minutes after the last viewer left, in a routine +on `host.references`. One fewer lifecycle state and timer in the kernel. +What breaks: nothing; the same behaviour moves to ten lines of app code. + +C8. Remove `state: false` (method-only references). +Why: B5; the snapshot is sent once per hold and the "shared store + view" pattern already keeps +big state out of the types that are mostly called. One hold kind, one `bound` shape. +What breaks: a consumer that only calls methods on a large-state object pays one snapshot. + +C9. Remove `previous` from derives. +Why: §12's rule is structural sharing through `mapShared`; `previous` is a second, name-keyed +cache across reloads that a derive "must be correct without". Two caches for one job. +What breaks: incremental derives that used `previous`; `mapShared` covers them. + +C10. Merge `constant`, `self.writable` and the in-process readable into one `cell(initial)`. +Why: `constant(v)` is a writable never set; both are "a readable of any in-process value, +identity-compared". One constructor; exposing a cell as a state member is allowed only when its +contract member has a schema (JSON), checked at implementation time (closes B10). + +C11. Remove `ctx.trace`; use `traceIn(ctx, …)` from `chord/context` (one spelling). + +C12. Remove call credits. +Why: §22 (R1) counts every message's parts in the byte window, calls included. A second credit +protocol for calls adds a frame kind and bookkeeping for nothing the byte window does not do. +What breaks: nothing; a flood of tiny calls still has header bytes. + +C13. Collapse the timeout table (§3) to two host options. +`closeTimeout` (native close, call drain, routine handover, job abort drain) and `readyTimeout` +(activation, awaited `get`, parked calls, rollout install). Per-thing overrides only where a case +appears. The `finish` backoff is the object backoff. + +C14. Remove `handle.change` and `handle.retire` shorthands. +Why: one way to write (`self.tx`); the shorthands are two more entry points to document and test. + +C15. Remove `inVariant`/`expectVariant` helpers from the kernel (keep `RefIn`); they are one-line +checks on `ref.variant` that apps can own. + +Keep as is: hosts/links/routes, principals and revocation, documents and store (incl. tags and +eviction), derives with `mapShared`, natives with close reasons, registries with runtime +`compose`, presence, R1 wire, telemetry. + +## D. Blockers and majors + +D1. major, §22 (R4) "an unknown hold or call id, is a protocol error that closes that link". +Trace: provider revokes a guest → sends `gone(h1)`; the tab's `call(h1)` (or `release(h1)`, +or `cancel(c7)` after `result(c7)`) is already in flight; provider receives an unknown id → +closes the link → every other hold of that tab drops and re-holds. Normal races become link +resets. +Proposal: `release`, `cancel` and `call` on an unknown id are answered, not punished: `call` → +`result` with `gone`; `release`/`cancel` → ignored. Only a malformed header closes the link. + +D2. major, §16/§17/§24: a plugin tool has no way to get a `ctx`. +Trace: the subagents plugin contributes a Durable tool `spawn`; its `execute` must call the +server's catalog (`Catalog.create`). Durable calls `execute(args, toolCtx)`; `host.root` exists +on the host object only (§3), which plugins never hold; `FacetApi` is unspecified beyond +`serve`. The first proof's tool with an approval pause is the harness's own; the second tool is +blocked. +Proposal: the facet's `host` (FacetApi) gets `root(fn, { principal?, signal? })`, and the core +harness passes a Chord call scope to every tool execution (`toolCtx.chord: Ctx`, host principal, +abort linked to the task), so a tool writes `await ctx.get(Catalog, {})`. + +D3. major, §22 (R2) "An oversized message fails only its own call or member with `too_large`". +Trace: one `state` message per commit per link carries parts for every hold the commit touched; +`maxPayload` (16 MiB) applies to the whole message. A chat window hold with a 20 MB snapshot or +delta in the same commit as five small holds fails all six; a `bound` snapshot above 16 MiB fails, +the consumer re-holds (R19 admits snapshots), fails again, forever. +Proposal: limits per part, enforced from the header before decoding (sizes are in the header); +an oversized part fails only its hold's member with `too_large`, the other parts apply; a hold +whose snapshot exceeds the limit is `failed` with `retryAt: undefined` (no automatic re-hold). + +D4. major, §13 (R16) vs. §13 close reason: R16 "disposes of it when its scripts finish or after +the default `closeTimeout` (30 s, no override)". §13: "end it when its running scripts finish or +hit their deadline". §24: "running scripts finish or hit their deadline". +Trace: a code mode script with a 5-minute deadline is 40 s in; the plugin is reloaded (the first +proof's "code mode reload"); after 30 s the detached sandbox is disposed; the script's nested +Durable call fails; the agent turn fails. +Proposal: the detached sandbox is disposed when its scripts finish or hit their own deadline; +`closeTimeout` bounds only `close` itself (which returns at once). Say so in §13 and R16. + +D5. major, §3 (B6): IPC flow control by frame count with unbounded part sizes. +Trace: a worker publishes 10 MiB snapshots (file index) to 20 relayed tabs behind a slow server +event loop: 20 parts per commit, window of N frames, memory grows by N × 10 MiB before +backpressure. +Proposal: one byte window for every link kind, counted from part sizes in the header (free on +IPC and memory now that parts are bytes or sized objects). + +D6. major, §19/§3: keep-alive inputs have no API. +Trace: the worker supervisor report needs "any client hold on this host", active calls, running +jobs. §19 gives `host.references(Type)` per type; a plugin would have to enumerate every type it +knows, and foreign plugins' types are unknown to it. +Proposal: `host.activity`: a readable `{ holds: number, calls: number, jobs: number }` for the +whole host (clients only for `holds`), plus `host.references(Type)` for presence per type. diff --git a/apps/p2/docs/p3-reviews/v2-review3-q10-prototype.md b/apps/p2/docs/p3-reviews/v2-review3-q10-prototype.md new file mode 100644 index 000000000..660257d12 --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review3-q10-prototype.md @@ -0,0 +1,72 @@ +# Q10 prototype review (contract evolution): Fable and Astra + +Prototype reviewed: `/tmp/evolve-proto/evolve.ts`, `evolve.test.ts` (forward evolution: `remote(name, fullV1)`, +`evolve(prev, e => e.state(…).method(…).removeMethod(…).addMethod(…))`, typed `Impl`, `AcceptsTable` keyed by +old version with mappings required at type level from the edit lists, `NoInfer` on `accepts`, runtime materialization, +normal-form fingerprint and completeness check at `serve`). Both verdicts: **the approach works and stays; the +prototype is not adoptable as is.** Chains of 5, 9, 10, 20 versions type-check in 0.11-0.14 s (TS 7) with readable +errors. + +## Fable (probes in `/tmp/evolve-probe`) + +Proven: type-level required mappings on the hard cases (shared sub-schema stated per member; a method changed in v2 and +removed in v3 needs only `call | "refuse"`; wrong-direction mappings are compile errors); exact old fingerprints by +materialization; the runtime check catches untyped providers by member name. + +Holes: +- Members added after v and then edited or removed are wrongly required for v (v4 `addMethod("editor","m4")`, v5 + `method("m4", …)`: `accepts[1]` demands `m4` typed `unknown`; add-then-remove demands `call | "refuse"` for a member + v1 never saw). Runtime does not require it: type and runtime disagree. Fix: filter edits by membership in the old + description (`Extract<…, keyof AllMethods>`, also for state and removed). +- Result-only or args-only change over-requires (both `args` and `result` demanded). Fix: runtime requires only the + changed part; the type keeps the method entry required with `args`/`result` optional per change. +- Changed and changed back (v2 Number, v3 String): the type demands an identity mapping, runtime does not. Harmless; + document or require `identity`. +- No edits for state add/remove, variant add/remove, input (direction (c) needs `e.input(schema)` = new base, older + accepts dropped). A removed state member or variant shows only as a runtime "missing mappings … (removed)" with + nothing the author can write. +- A method cannot be added to a second variant (`addMethod` refuses any existing name across variants). +- D11 unenforced: `remote()` accepts `f: method(String)` in variant a and `f: method(Number)` in b; `Impl` becomes + `(x: string | number)`. +- Normal form unstable: `required` not sorted (reordering properties changes the fingerprint), `anyOf` order likewise; + `examples` kept. TypeBox 1.3: `Optional` only through `required`; `Literal` → `const`; `Record` → + `patternProperties`; `Cyclic` → `$defs/$id/$ref` (renaming `$id` changes it); `Unsafe` → whatever the author wrote. +- Mutation after declaration: schemas shared by reference; mutating one leaves the stored fingerprint stale while the + completeness check compares the mutated objects. +- JavaScript slip-throughs: `accepts[1].methods.r = "nonsense"` accepted (only `!== undefined` checked); unknown + mapping keys ignored; a hand-built `Contract` with a forged `history` is trusted by `serve`. + +Smallest changes: (1) filter required mappings by membership in the old description; (2) `args`/`result` separately; +(3) add edits `addState`, `removeState` (mapping current → old member or `"refuse"`), `addVariant(name, { as })`, +`removeVariant` (refuse), adding an existing method to another variant, `input(schema)` as a new base; (4) enforce D11 +in `remote()` (type and runtime); (5) normal form: sort `required`, `enum`, `anyOf`/`oneOf` by canonical string; drop +`examples`, `$comment`, `deprecated`; keep `default`, `format`, `pattern`, bounds; freeze with the protocol version and +pin TypeBox; (6) deep-copy or deep-freeze schemas into the history; brand `Contract`; (7) runtime shape checks on +`accepts` and reject unknown keys. + +## Astra + +1. **Blocker:** `normal()` drops `title`, `description` and `~*` everywhere, including property and member + dictionaries: `Type.Object({ title: Type.String() })` and the Number equivalent have the same fingerprint, and the + change bypasses mapping completeness. Unsorted `required` gives false mismatches on reordering. +2. **Major:** historical schemas are mutable aliases (see Fable). +3. **Major:** the runtime check tests presence, not validity: state mapping `null` or `42`, changed-method mapping + `{}`, removed-method mapping `0` pass. Inconsistent method signatures across variants are accepted; `addMethod` + permits a method with an existing state member's name. +4. **Major:** accumulated edit unions include members the old version never had (V2 adds `temporary`, V3 removes it; + accepting V1 requires a `temporary` mapping typed `unknown`). Changed → removed and one method in two variants + work. +5. Gaps: variant `"v.x"` becomes `state.v["x.s"]` (edits encoded as dotted strings); adding an existing method to + another variant fails with `never`; state removal and variant add/remove are absent and rejected at runtime even + with mappings; the base version is fixed at 1, so dropping the bottom of the history cannot keep published + version numbers. +6. Conservative, not unsound: result-only changes require an identity args mapper; no-op edits require identity + mappings. +7. Scaling fine (5/10/20 versions 0.11-0.14 s); Optional, Union, Literal, Record, named `Cyclic` survive + fingerprinting; executable refinements via `Unsafe` are silently dropped from fingerprints instead of rejected. + +Smallest changes: a schema-aware, data-only normal form (schema nodes vs property dictionaries vs literal data; +reject executable features); immutable schema and history metadata owned at declaration; structured edit records, not +delimiter-encoded strings, with obligations from historical membership and current disposition; runtime validation +of mapping shapes, edit targets, member-kind consistency and shared method signatures; explicit version ids per +history node (rebasing); the missing structural operations, or an explicitly excluded subset. diff --git a/apps/p2/docs/p3-reviews/v2-review3-summary.md b/apps/p2/docs/p3-reviews/v2-review3-summary.md new file mode 100644 index 000000000..89e848ef4 --- /dev/null +++ b/apps/p2/docs/p3-reviews/v2-review3-summary.md @@ -0,0 +1,66 @@ +# Chord v2 review round 3 (final): summary for the owner + +Sources: `v2-review3-fable.md` (F), `v2-review3-astra.md` (A), both read from `origin/private` at `138268fa3`/`1fbadcf1e`. +Verdicts: Fable "up overall, buildable after two majors; not yet minimal"; Astra "down for final sign-off: 1 blocker, +6 majors". Nothing here is decided. + +## 1. Text fixes (no decision: they apply existing decisions) (done, owner OK 2026-10-19) + +| # | Fix | Source | +| --- | --- | --- | +| T1 | §3, §5 table, §27 still say "only untrusted callers are validated": apply R7 everywhere | F B1, A B1 | +| T2 | §24 still says the harness's activation is the start barrier: apply R17 | F B2, A B2 | +| T3 | §24 example passes `registry` once: pass a function reading the composed registry, like `models` | F B3 | +| T4 | §9 `Settings` and §15 `ShellResult` examples lack the required `checkpointWhen` | F B4, A B5 | +| T5 | §9 "handles may be used in transactions of their own object (or of the facet that serves it)" reverses the facet rule | A B4 | +| T6 | §21 "the registries hold everything that will be there" vs. facets retrying with backoff: soften (late contributions recompose; R17 covers it) | F B7 | +| T7 | §30 cost guard "one encode per link": count part encodes at the origin, none at relays | A B9 | +| T8 | §15 job outcome and §20 flow state are not tagged by `kind` (rule 6) | A B10 | +| T9 | §31 rejects "validating state by walking values" but R8 validates migration output once: narrow to commits | A B11 | +| T10 | §6 "same documents and natives" for pure objects (they have no natives) | F B13 | +| T11 | log entries superseded but unmarked (broken dependency, staging conflicts, per-module digest, copying arguments, `Context.telemetry: TelemetryContext`) | F B8, B12 | + +## 2. Decisions, one at a time (blocker and majors first) + +| # | Sev. | Problem | Proposals | +| --- | --- | --- | --- | +| Q1 (decided: ignore unknown ids on both sides) | blocker | "Unknown id closes the link" (R4) turns normal races into link resets: a `result` crossing a `cancel`, a `call` crossing `gone`, a `release` crossing a state part (F D1, A D1) | F: answer, don't punish: `call` on an unknown hold gets `result` with `gone`, `release`/`cancel` on unknown ids are ignored, only a malformed header closes. A: retiring states with terminal acknowledgements, distinguishing retired ids from never-issued ones, bounded bookkeeping | +| Q2 (decided as proposed) | major | Size limits (R2): `ws.maxPayload` closes the whole WebSocket (1009), so it cannot fail one call; one `state` message carries many holds' parts, so one big part fails all; a relayed compressed part is decoded only in the worker, where IPC has no limit (F D3, A D2) | Limits per part, from the header sizes, before decoding: an oversized part fails only its hold or call; a snapshot over the limit fails that hold without automatic re-hold. The decoded limit travels with relayed work (stamped at admission), so the worker enforces the client limit. `maxPayload` stays only as the transport's hard cap (closes the link) | +| Q3 (decided as proposed) | major | R19's re-hold waits for credits covering the snapshot size, which can never happen for a snapshot larger than the window: permanent stall (A D3) | Re-hold once the previous snapshot's debt has drained (credits back to the full window), not once credits exceed its size; keep the minimum interval | +| Q4 (decided: one binary format and byte window on every link) | major | IPC flow control counts frames, but R1 parts are bytes: memory unbounded in bytes (F B6, D5) | One byte window for every link kind, from the part sizes in the header | +| Q5 (decided: rejected; activation chain of instance ids across hosts) | major | Dependency cycles: with R18 failed dependents keep their references, so A↔B never lose their last reference (A D4) | Reject declared dependency cycles before activation waits (also cycles introduced by a reload), as a broken declaration | +| Q6 (decided: host under a context key, `chord(context, fn)`) | major | Plugin tools in Durable cannot get a Chord `ctx`: `host.root` is on the host object, which plugins never hold (F D2) | The facet's `host` gets `root(fn, { principal?, signal? })`; the core harness passes a Chord call scope to every tool execution (host principal, abort linked to the task) | +| Q7 (decided: R16 stands, 30 s at most) | major | R16: a code mode script with a 5-minute deadline, 40 s in at a reload, is killed at 30 s; its turn fails (F D4) | Dispose of the detached sandbox when its scripts finish or hit their own deadline; `closeTimeout` bounds only `close` (which returns at once). Owner said 30 s is plenty; the reviewer reopens it with this trace | +| Q8 (decided: one declaration from the defining facet; accepts stays; mismatches left out and reported; use sites guard) | major | Registry definition ownership on reload: plugin A reloads registry R with a new `compose`; unchanged plugin B still imports the old token (A D6) | The defining facet holds the one authoritative declaration; other tokens only identify the registry; a reload replaces the declaration atomically | +| Q9 (decided: unsettled runs only; the owner dedupes after completion) | major | Job dedupe and record retention have no end: 100,000 keyed jobs stay in memory, or a retried key reruns its effect (A D7) | Define the dedupe horizon (only unsettled runs, or retained completed history) with record retirement; or shrink/defer jobs (C3) | +| Q10 (decided: forward evolution in code; implemented `7d2bb8ea8`) | major | `accepts` keeps old member schemas but the old fingerprint covers the old input too: an input change cannot bind old holds (A D5, B8) | Prohibit input changes through `accepts` (a new object type instead); or remove `accepts` entirely (C2) | +| Q11 (decided: singletons always have a value) | minor | Singleton existence: §9 says an absent document reads `undefined`, also "a singleton that does not exist yet reads `initial()`" (F B11, A B3) | Singletons read `initial()` (never undefined); `undefined` only for absent keyed documents; singleton existence needs no API | +| Q12 (decided 2026-10-21: closed schemas, unknown fields rejected) | minor | Stripping arguments (R7) vs. "nothing is copied": stripping a shared object must not delete fields from it (A B7) | Copy on change: a stripped argument is a new object only when it had extra fields; never mutate the caller's value | +| Q13 (decided 2026-10-21: relays stamp the origin incarnation) | minor | `ctx.incarnation` on relayed calls is undefined: the worker's handshake is with the server (F B9) | Relays stamp the origin host's incarnation, as they stamp the principal | +| Q14 (decided 2026-10-21: only JSON readables back state members) | minor | A writable holding functions could be exposed as a state member and reach the encoder (F B10) | Only members with a JSON schema in the contract are exposable; checked when the object is served | +| Q15 (decided 2026-10-21: local arguments validated too) | minor | `local` arguments are "neither validated nor copied" in §5's table, but R7 says every caller (A B1) | Apply R7 to `local` too, or keep `local` as the one exception | + +## 3. Primitives to remove or merge (owner choices) + +| # | Cut | Source | What breaks | +| --- | --- | --- | --- | +| C1 (rejected, owner: `local` stays its own kind) | merge `local` into one object kind with `principals: false` | F C1 | nothing in p3 | +| C2 (moot: `accepts` stays, third-party plugins are a goal) | remove `accepts` for object types and registries (and adapted-binding diffs); strict fingerprints only, since hosts install before anything is routed | F C2, A C6 | a plugin built against an older contract fails to bind instead of being adapted | +| C3 | shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel into an optional component | F C3, A C1 | `defineJob` loses versioning; or core-only users lose `self.job` | +| C4 | routines: `on` and `loop` only (`every` a helper; a due time that must survive is a document) | F C4 (A: keep four forms over one runner) | `every`'s due time across reloads | +| C5 | merge `lifetime: "host"` objects into the facet (the facet may open natives and contribute); every served object is referenced | F C5 | a plugin's always-on natives share one failure domain | +| C6 | remove `broken()` for opens and `retryOn` (bad configuration retries with the backoff) | F C6 | nothing; reload and Retry still work | +| C7 | remove provider-side `linger` (app keeps a hold for N minutes) | F C7 | nothing; ten lines of app code | +| C8 | remove `state: false` (method-only references) | F C8 | a method-only consumer pays one snapshot | +| C9 | remove `previous` from derives (`mapShared` covers it) | F C9 | derives that used `previous` | +| C10 | one `cell(initial)` for `constant` and `writable` | F C10, A C3 | nothing | +| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | F C11 | nothing | +| C12 | remove call credits (the byte window covers calls) | F C12 | nothing | +| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | F C13 | per-thing overrides | +| C14 | remove `handle.change`/`handle.retire` shorthands | F C14 | nothing | +| C15 | remove `inVariant`/`expectVariant` (keep `RefIn`) | F C15 | nothing | +| C16 | remove facet-owned mutable documents and the two-line transaction exception (shared data in an ordinary local object) | A C2 | atomic writes across a served object and its facet move to one owner | +| C17 (decided 2026-10-21: accepted, log "The harness is an ordinary object") | no harness-specific barrier exception: the harness is an ordinary object the app's start code holds after `load` | A C7 | pre-barrier factories cannot depend on the harness | + +Implementation notes (no decision): one subscription substrate for writable, constant, derives and registry values +(A C3); one runner for routines (A C4); one lease implementation behind `host.get`/`hold`/`self.get`/`ctx.get` (A C5). diff --git a/apps/p2/docs/p3-walkthrough.md b/apps/p2/docs/p3-walkthrough.md new file mode 100644 index 000000000..878e77741 --- /dev/null +++ b/apps/p2/docs/p3-walkthrough.md @@ -0,0 +1,317 @@ +# p3 / Chord v2: the whole target design as a walkthrough + +A reading aid for checking completeness, not a new source. Sources: `redesign-decisions.md` (wins on conflict), +`target-design.md`, `questions.md`, `plugin-loading.md`, `p3-functions/`, `changes.md`. Marks: **[D]** decided, +**[B]** built in today's Chord, **[S]** superseded by a later decision, **[O]** open, **[P]** proposed, not decided. +Section 17 lists conflicts between the documents found while writing this. + +--- + +## 1. Plan + +- **[D] p3 next to p2**, as `apps/p3`, own data directory. p2 keeps running unchanged. Plugins are ported one at a time. +- **[D] No migration** of p2 data: p3 starts empty (no `seed`, no catalog/shares/subagents import, no legacy parsing). +- **[D] Chord v2 lives inside p3** until stable, then moves to `packages/chord` (pi first, CI) and p2 is deleted. +- **[D] Durable boundary:** Durable keeps importing today's Chord (`delta`, `context`, `output`); p3 hands Durable only + plain data and Durable's own types. Chord v2 takes `delta`/`output` over unchanged. `OutputBuffer`/`Progress` already + moved to `@earendil-works/chord/output` [B]. +- **[D] No backward compatibility** anywhere unless asked. +- **[D] Approach:** functionality first (`p3-functions/`: 22 domains), then objects per domain (owner, host, + pure/native, relationships), then target design v2 written from the decision log, then reviewers judge it. + +## 2. Concepts + +| concept | what it is | +|---|---| +| facet | a plugin's code for one host kind; its setup is a **pure** host-lifetime object [D] | +| object type | contract: name, input (address), variants, version, host [B] | +| object | one instance per (type, input, principal key) | +| principal | who calls; stamped by links, typed by the app [B] | +| reference | what `get` returns; keeps the object alive; live and hydrated [B] | +| document | named JSON value an object owns, written in transactions | +| derive | value computed from documents/readables | +| native | a value opened with `self.open` in an object (socket, process, DB handle) [D] | +| routine | standing behaviour, `self.routines.*` | +| job | owned work started by a method, outlives the call | +| registry | an owner's extension point, versioned immutable table | +| host value | typed native the host provides before facets load | +| `ctx` | per call: signal, principal, call-scoped `get`; in jobs `jobId`, `progress` | + +## 3. Object types and principals [B] + +- `local` (same host, unvalidated) or `remote` (over links, variants per principal, TypeBox-validated). +- `host` declares placement (`hostNamed`, `hostFamily`); changing it is breaking. Callers don't pass `at`. +- Same version binds if the provider covers the caller; breaking change bumps `version`; `accepts: { [old]: adapter }` + keeps old consumers, else `contract_mismatch`. +- References are a union discriminated by `variant`; strict narrowing with `inVariant` / `expectVariant` / `RefIn` + (entry 8). Weakening type checking was rejected. +- Principals: `createHost

()`, every principal has a `kind`. p2: `owner` (`access: local | relay | device`), + `guest`, `worker`, `server`, `host`. State `init` (2b). Origin ids per client (entry 8). Protocol 4. +- `host.serve(Type, factory, { lifetime: "referenced" | "host", linger? })`; `"host"` checked by types [B]. + +## 4. Factories, natives, the facet [D] + +- **Factory principle:** a factory is synchronous and only declares (references, documents, natives, derives, + routines, jobs, registrations). Chord activates. Handles it gets are valid for the object's whole life. +- **Natives only in objects:** `self.open(async ({ signal }) => value, close)`. Chord opens during activation (object + goes live after all opens), closes in reverse order when the object ends. The value never changes during the + object's life. + - Gone [S]: `self.resource`, resource `version`, `replace`, `deps`, reopen, stale handles, "read `.value` at use" + (target-design §12). +- **No natives across plugins:** others reach a native only through the owning object's methods and state. +- **The facet is a pure object:** documents, references, routines, registrations; no `self.open`. Served objects may + read/write the facet's documents; one transaction may span own docs + facet docs (lock order: own line, then + facet). The facet may not reach into served objects' documents. +- Between plugins only: object types, registries, host values. +- Rejected: ceremony (`native/` dirs, `defineNative`, lints). + +## 5. Ownership [D] + +- Every thing has exactly one owning object, the smallest it belongs to. The owner holds the data, enforces rules in + its methods, offers operations. Nobody reaches past it. +- Ownership nests: session owns its conversations and location; a conversation owns its content and rules (read-only, + follows session cwd), set in the creating commit (Durable `createConversation` with `init`). +- Methods return data; objects are handed out as addresses (`ctx.get(Type, address)`). +- Registries add behaviour (tools, routes, renderers, commands, send routers, input gates); never guard owned data. +- Core owners always exist; plugin owners may be missing (→ optional references [O]). +- Avoid a core god object. +- **Porting guideline:** what core enforces on data lives in a format core owns, written by the plugin at creation. + Example: one-shot / cwd-following children marked in a core per-conversation policy document; `ReadOnlyConversations` + and `SessionCwdFollowers` go away. + +## 6. References + +- [B] `self.get` (factory, released with the object), `await ctx.get` (call), `host.get`/`host.hold` (outside). +- [B] Awaited reference is live and hydrated; `state: false` for method-only use. +- [B] Link drop: values stay (`loading`), re-hold automatically, calls wait. +- [D] Failed object ends its references with the cause; consumers re-hold with backoff (web gets the TUI's backoff + instead of fixed 500 ms). +- [D] **Backoff per address** (1 s doubling to 60 s): a `get` during backoff returns the stored error; concurrent gets + share one attempt. Cut short by: plugin reload, a dependency of the failed attempt changing, `ref.retry()`, + `self.retryOn(...readables)`. +- [D] Consumers handle errors as on a disconnect. No call/request-id framework (Chat's `requestId` stays). + +## 7. Documents and transactions [D] + +- `defineState(name, { schema, version, initial, migrate?, keyed?, persist?, checkpointWhen? })`. +- `self.state(Def)` → one handle. Singleton: loaded at activation, `handle.value` sync, exposable readable. Keyed: + `await handle.get(key)`, `for await (key of handle.keys())` (paged from storage). No live key list; live collections + are singleton docs; large ones are paged methods. +- Writes only via `await tx.draft(handle, key?, { tags? })` in `self.tx(ctx, async tx => …)`; `tx.retire`. +- **Durable's rule:** inside a tx only `await tx.*`; no other objects, models, processes, network, nested tx. Fetch + before; decide inside. +- Commit: per-object line; storage batch serialized per host; adopt; notify once; one wire batch. +- Shorthands: `handle.change(ctx, key?, fn)`, `handle.retire(ctx, key?)`. +- Versions: pure `migrate`, on access; old code on newer doc throws; no `migrate` → access refused, data kept. +- Tags: typed (`defineTag`), set explicitly on create; `store.retireByTag`. p2 `SessionTag`. +- Lifetime: in-memory docs live with their object identity; persisted until retired / session delete. +- `persist` only on stable identities (`lifetime: "host"` or `storeKey(input)`). +- Rejected: optimistic transactions; Durable's Session as object store; volatile documents. + +## 8. Store and storage [D] + +- Chord owns the store: lines, serialized commits, migrate on access, `checkpointWhen`, retire, tags, invariants. +- Eviction as in Durable for keyed persisted docs (10 min unused and unwatched; LRU, one timer). +- Storage error ends the store; host reports and restarts. +- `DocumentStorage` interface (`commit`, `find`, `load`, `scan`, `byTag`, `mintId`); memory implementation in Chord; + p2/p3 adapts Durable's SQLite `Storage` (`kind = .`); tag index in the adapter. +- Conformance suite in `@earendil-works/chord/testing`. +- Persistence split: app state in Chord documents; only transcript-bound data in Durable. +- Kinds of state: document, derive, source readable, paged method, plugin database (as a native). + +## 9. Derives [D] + +- `self.derive(name, sources, (values, previous) => …)`; `previous` kept across factory re-runs by name. + +## 10. Jobs [D] + +- `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`. +- Sandwich: **intent** `tx.start(job, args, { key? })` (busy check and dedupe atomic, run starts after commit) → + **run** `run(args, ctx)` (anything; external writes keyed by `ctx.jobId`; keeps old code on reload) → **finish** + `finish(outcome, tx)` in current code, ack in the same commit; retried with backoff if it throws. +- Outcome `{ ok, result } | { ok: false, error: threw | aborted | lost }`. +- `job.start(args, ctx, opts)` returns a run with `outcome(ctx)`; caller's abort ends only its wait. +- Abort is a request (`job.abort(id)` fires the signal). +- `running` readable `{ id, version, args, progress, phase }`; `concurrency` counts active runs; `phase: finishing | + blocked`. +- Versions via `accepts`; stored outcomes never rewritten; missing adapter → `blocked`, not a refused reload. +- `persist: true` (stable identities): a live run at crash ends `lost`. +- Running jobs count for keep-alive and the relocation fence. +- Not jobs: request/response (method), input into running work (Questions), crash-resumable work (Durable task). +- [O] jobs as objects with an "until done" lifetime. + +## 11. Routines [D] + +- `self.routines.on(readable|event, fn)`, `every(name, interval, fn)` (due time survives reloads), `loop(name, fn)`, + `custom(name, setup)`. +- Start when the object goes live, stop when it ends, restart on reload of its plugin; serialized per routine (new + instance waits for the old handler in flight, with a deadline). +- Reconcile from current value; no "in progress" flags; long work → job. +- No bare timers/callbacks outside routines, jobs or calls. +- [O] `overlap: "latest"`. + +## 12. Registries and host values [D] + +- `defineRegistry(name, { version, accepts, conflict: "error" | "last" })`; facet registers; entry carries its + build version; tables are immutable values swapped atomically; no late binding; no lifecycle in entries. +- `hostValue(name, { version })`, provided before any facet on every host (worker: harness, environments, init, + location, liveness, cwd followers; server: session hosts, server info, revocation; web/TUI: navigation, theme, + clipboard). Replace `opaque` members. +- **Start barrier:** open Durable without scheduling → host values → load facets → check cwd/model → `resume()`. +- Rejected: late-bound entries, dynamic registries. + +## 13. Reload [D] + +- Explicit (`npm run reload -- `), never on save. Behaves like a restart of the changed plugins. +- **Per host:** load (bundle; error rejects) → **staging** (facet setup and every live object's new factory with a + recording `self`; nothing activated; a throw rejects) → **commit** (atomic) → after. +- **Rollback all-or-nothing per host:** any load/staging error leaves the host entirely on previous code. Status per + host suffices. +- **Pure objects** swapped in place: new methods and derives; documents, references, jobs stay; routines restart + serialized; new references go live; gate opens. Running calls finish on old code. +- **Native objects of the reloaded plugins restart:** running work aborted (`aborted`), natives close, factory runs, + natives reopen. No per-module digest; split expensive natives into their own plugin. Open fails → that object + `failed` with backoff; rest of host on new code. +- **Remote consumers** see a normal brief object restart: `loading`, values readable, calls park, re-hold; calls on the + old instance end with an error. +- Errors: code errors (load, setup, factories) → rollback; activation errors (opens, new references, migrations) → + per object; a method bug → that call. +- No factory trial, no migrate dry run, no consumer contract check during prepare. +- Removing a type: its objects end; in-memory docs go; persisted docs stay. +- Rejected: drain-based handover, version fences, `restart_required`, reload votes / cross-host transactions, + revoking proxies on `.value`. + +## 14. Rollout [D] + +- Revision = (server incarnation, sequence) over the complete desired code set (every facet + digest). +- Server installs, then each worker, then clients; each reports `installed` / `installed_failed` / `rejected`. A + starting or reconnecting host installs the current revision before anything is routed to it. Installed = code + published (objects activate on demand). +- Mismatch responses carry the provider's revision and status: provider ahead or installing → `updating` (values + readable, calls park); provider rejected → fail fast; same revision → `contract_mismatch` final. Deadline on waits. +- Server rejects → rollout stops. +- [O] check own host's status before waiting; serialize installs per host; digest of the code set (no reinstall after + a server restart). + +## 15. Wire, presence, Questions, ctx + +- **Wire [D]:** documents are state members; `snap` on hold, `ops` per commit, diff ops for derives, throttled job + progress, `reset` for migration, `error`. Commits atomic for consumers: one `state` frame with a `batch` per hold. + [O] atomic per link, diffed ops for adapted bindings, size limit. +- **Presence [D]:** `host.references(Type)` readable (input, principal, clients); origin ids per client; **linger bit** + (lingering keeps the worker alive, not a viewer). Replaces presence/viewer/login-abandon leases. + [O] link drop counts as lingering; counts per origin. +- **Questions [D]:** p3 service for input into running work: shared store, per-principal asker/viewer views, typed + questions (text, select, confirm), policies first/quorum/all over a live audience, `key` to reattach, deadline, + authorization per question, secrets redacted. Used by login, tool approval, question tool. +- **ctx [D]:** `signal`, `principal`, `get`; in jobs `jobId`, `progress`. No code versions, no root ids. + +## 16. Bytes and session files + +- **[D] Bytes are a schema type** (`Uint8Array`): structured clone on IPC (`serialization: "advanced"`), binary frames + on WebSockets. No base64 in JSON. Lives in Chord v2 inside p3 (not added to today's Chord). +- **[D] Large transfers chunked by the owner:** `begin` / `append({ upload, offset, bytes })` / `finish`, + `read({ id, offset, length })`, ~1 MiB chunks; awaited `append` is backpressure; resume from stored offset after a + link drop; unfinished uploads expire. Helper wraps the loop. No stream type in Chord. +- **[P] `SessionFiles {session}`** on the server: one owner of uploads (25 MiB, ≤20/message), sent-file snapshots + (25 MiB), artifact versions (2 MiB, name + version); index readable without a worker; bytes under + `/sessions//files/`; copy on clone and fork, delete with the session. Not in it: workspace downloads + (streamed), shell/codemode spill (in the env). `DownloadStore.save` (unused) goes. + Pending sub-points: one preview limit (256 vs 512 KiB); fork's artifact list empty or copied; copying attachments + into the remote env. + +## 17. Session lifecycle [D] + +- Archive retires nothing (stops the worker, ends guest access, closes forwards). +- Session delete (new) removes the whole session: tagged documents, the worker's database and document storage, + files, catalog entry. [O] how: `SessionDeleted` registry vs. per-session objects + `SessionTag` + session directory; + search index per session vs. global + hook. + +## 18. Versions and adapters + +| surface | on change | +|---|---| +| object type | `accepts` adapter, coverage check [B] | +| registry | `accepts` adapter | +| host value | `accepts`; harness refused on mismatch for now | +| document | pure `migrate` | +| job | `accepts` on outcomes and `running` | +| native | object restarts on reload (no version) | +| Durable tasks/docs | Durable `migrate` [B] | +| link protocol | refused; hosts update together [B] | +| UI components, API | exposed as they are; versioned with the SDK later | +| [O] file formats (`envs.json`, …) | unversioned today | + +## 19. Plugin loading (`plugin-loading.md`, revision 4) + +- Sources: built-in, global (`/plugins`), global packages (npm/git, `--ignore-scripts`), project + (`/.pi/p2/plugins`, worker/web/tui only, never server). +- Discovery = directory listing, never parses code; server selects, workers scan nothing; dependencies resolved by + contract name at runtime; missing provider fails at activation with the consumer's name. +- Names: `.` for contracts, registries, Durable extensions, entry kinds; tools `_`. +- Loader D: one base bundle per surface + one script per facet, host packages passed in, no module map (GC reclaims old + generations); stable filename per (facet, digest), V8 compilation cache off. Vite removed after web D. +- Project trust: `trust.json` per directory, `ask`/`never`, pin-to-content mode; content-addressed snapshots; approved + bytes = executed bytes; change → dirty → owner Reload. Project `.pi/skills`/`.pi/prompts` gated too. +- Scopes: a page/TUI worker holds one scope; `ClientBundle` pushes bundle hashes; TUI snapshots partitioned per scope. +- Leaving a scope: provenance map, fallback renderers, blocked tasks with Abort. +- Security table: trusting a project grants the owner's OS account and owner rights in the browser. +- Contracts are a versioned package, never host packages. +- [O] base skills on remote sessions; publishing; plugin identity across projects; guest opt-in; project root; browser + compile cache. Project plugins as a whole: separate discussion. + +## 20. Functionality map (`p3-functions/`) + +22 domains; cross-cutting needs (one file store, per-session lifecycle, readers without workers, generation-fenced +reports, catch-up after commits, work outliving its caller, request-id idempotency, cancellation chains, keep-alive vs +presence, execution env following the session, single config writer, attribution, one permission matrix, revocation, +chunked bytes, one notice channel, UI state survival, TUI consent); ~25 duplicates; ~18 drop candidates. + +## 21. Open (all) + +- Optional references (a plugin owner may be missing; radius dies without ports today). +- Keep-alive computed by the worker itself. +- UI renderer reload vs. leaving (login aborted, TUI overlays close). +- File formats as versioned interfaces. +- Terminal host process as a revision artifact. +- Durable documents shared across plugins. +- Session delete mechanics (section 17). +- `close` with a reason (`reload` / `end` / `stop`). +- Call-scoped natives. +- Async activation guard for authority-carrying addresses. +- Invariants across several owners. +- Published projections with a generation fence. +- Connection pools; config files with a single writer. +- Rollout fixes (section 14); wire; presence; `overlap: "latest"`; project plugins; jobs as objects. +- Step 2 objects for every domain, starting with `SessionFiles`. + +## 22. Tracked, not scheduled (`changes.md`) + +- Leaks C3 (Chord kept listener), C5 (p2 guard listener per reload), C7 (harness keeps envs `close`); rerun GC tests. +- p2 hazards on interrupted calls (shell timer, dedupe in a closure, withdrawn text only in replies, web send while + disconnected, missing request ids, dropped terminal keys, multi-step revocation/relocate/clone). +- Durable nested commits hang by convention only. +- #10959 (remote references accept a foreign ctx), owner fixes via subagent. + +## 23. Rejected (all) + +Drain-based handover; version fences without transactions; object restart for state changes; Durable Session as object +store; volatile documents; per-group lines; generic call/request ids; optimistic transactions; outbox for transient +delivery errors; revoking proxies on `.value`; late-bound registry entries; dynamic registries; `restart_required`; +detached references counting toward presence; contracts taken from the running p2; ceremony (`native/` dirs, +`defineNative`, lints); weakening type checking; "magic" input-field matching for delete; factory trial; migrate dry +run; consumer contract check during prepare; reload votes; `defaultProjectTrust: "always"`; loader variants A and B. + +## 24. Conflicts found between the documents + +1. `target-design.md` §15 and `plugin-loading.md` 7.5: "a facet whose setup throws keeps its previous version while the + rest of the host moves on". The decision log replaced this with all-or-nothing rollback per host. +2. `questions.md` keeps its store in a facet `self.resource`. Facets may not open natives now, and `self.resource` is + gone: the store must move into an object (e.g. a host-lifetime `QuestionStore` with `self.open`, or plain documents + if in-memory is enough). +3. `target-design.md` §5 example (`self.resource` in the facet, `deps`) and §12 (resources replaced in place) are + superseded by `self.open` in objects. +4. `target-design.md` §7 "handles belong to their code generation ... `stale`" assumed in-place resource replacement; + with pure swap and native restart, whether old-generation handles still need `stale` is unclear. +5. `plugin-loading.md` 9.2a gap list mentions `opaque` members and unversioned registries; both are addressed by host + values and versioned registries. diff --git a/apps/p2/docs/plugin-hooks.md b/apps/p2/docs/plugin-hooks.md new file mode 100644 index 000000000..a7c12ba83 --- /dev/null +++ b/apps/p2/docs/plugin-hooks.md @@ -0,0 +1,456 @@ +# Plugin hooks + +**Outdated: describes p2 on Chord v1** (before the move to Chord v2; Chord v1 is deleted, so its APIs such as +`self.register` and objects of lifetime "host", and the `packages/chord/src/old` paths cited below, no longer exist). +The registries and hooks are largely the same on v2; how p2 uses Chord v2: `p2-notes.md` **Chord v2**. + +Reference of everything a p2 plugin could hook into, per facet. Paths are relative to `apps/p2/` unless they +start with `packages/`. "Owner/guest" says which principals reach a hook's effects; facet code itself is trusted code +that the server or the visitor's process runs. + +How plugins are loaded (and the planned single pipeline) is in [plugin-loading.md](plugin-loading.md). How to write +one, step by step: Chord's [tutorial](../../../packages/chord/docs/tutorial/). TUI design: [p2-tui-port.md](p2-tui-port.md) §6-7. + +## 1. Cross-cutting + +### 1.1 Facets and how they load + +| Facet | File | Loaded by | Runs as | +| --- | --- | --- | --- | +| Server | `plugins//server.ts` or `server.tsx` | Chord `nodeLoader` via `PluginRuntime.loader` (`src/hosts/plugins.ts`), alphabetical order | server host, `host.name === "server"` | +| Worker | `plugins//worker.ts` or `worker.tsx` | same, in every session worker, from the list in its init (`src/hosts/worker.ts`) | `session:` host | +| Web | `plugins//web.ts` or `web.tsx` | `buildWebBundle` bundles selected facets; `startWeb` installs them (`src/hosts/bundles.ts`, `src/hosts/web/main.tsx`) | browser client host | +| TUI | `plugins//tui.ts` or `tui.tsx` | `buildTuiBundle` bundles selected facets; `setupTui` installs them with the core facet (`src/hosts/bundles.ts`, `src/hosts/tui-worker-runtime.ts`) | locked-down worker on the visitor's machine | + +A facet is `defineFacet((host) => …)`. There is no manifest and no derived dependency list: a plugin needs what it +holds, resolved by contract name at runtime (`docs/plugin-loading.md` 4.2). + +### 1.2 `contribute()` and registries + +`contribute(host, name, register)` (`src/runtime/contribute.ts:10`) serves a local object of lifetime "host" whose factory only +calls `self.register(Registry, { key: value })`. Registrations live exactly as long as that object: a reload replaces +them, removing the plugin removes them. Only objects of lifetime "host" may register (`packages/chord/src/old/object.ts`). + +```ts +export default defineFacet((host) => + contribute(host, "p2.web-envs", (self) => self.register(Slots, { "sidebar-foot:envs": EnvsButton })), +); +``` + +A plugin that needs state, resources or other objects serves its own object of lifetime "host" and registers from its factory +(e.g. `plugins/codemode/worker.ts:39-60`, `plugins/terminals/server.ts:10-30`). + +Registries are plain descriptors (`defineRegistry`, `packages/chord/src/old/registry.ts:11`): + +- **Conflict mode.** `"error"` (default) or `"last"`. Every p2 registry uses `"error"`; `TuiSlots` and + `SessionArchived` get it by default. Two objects registering one key make the table build throw + `registry_conflict` (`packages/chord/src/old/host.ts:608-623`), which refuses the load or reload. `"last"` is unused. +- **Order.** A table iterates in host-wide first-registration order of its keys, stable across reloads + (`packages/chord/src/old/host.ts:625-638`). "Ordered" registries sort keys instead: `ordered()` with numeric-aware + compare and `:` keys (`src/runtime/tui-renderers.ts:48`), `ComposerExpanders` by key + (`src/runtime/composer.ts:10`), TUI prefix slots by key (`src/runtime/tui.ts:266-271`). Web `SlotList` uses + registration order (`src/runtime/web.tsx:77-88`). +- **Where defined.** Runtime registries live in `src/runtime/*`; plugins define their own in `contract.ts` + (`SendRouters`, `TranscriptEntries` in `plugins/chat/contract.ts:256,270`; `SessionArchived` in + `plugins/sessions/contract.ts:58`). Using one imports that plugin, i.e. depends on it. +- **Reading.** `host.registry(R).table(ctx)` / `.get(key, ctx)` inside calls, `.subscribe(fn)` for live tables; + web: `useRegistry(R)` (`src/runtime/web.tsx:52`). + +### 1.3 Contracts and references between facets + +- `local(name, …)` types are host-internal (only facets of the same host get them); `remote(name, …)` types are + reachable over links, with one **variant per principal** chosen by the factory (e.g. `Chat` viewer/participant, + `plugins/chat/contract.ts:195-243`). A factory that throws `forbidden()` denies. +- Placement: every remote contract says where it lives, `host: serverHost` or `host: sessionHost` + (`src/runtime/chord.ts`; `sessionHost` places an object on `session:` + its input's `session`). Callers never name a + host: a `get` goes where the contract places it, and the server routes `session:` to the worker, spawning it. Web and TUI hold server and worker objects; workers hold server objects with a + worker principal. +- References: web `useObject` (re-gets after `gone`, `src/runtime/web.tsx:139`); TUI `TuiScope.watch` (backoff, + stops on permanent refusals, `src/runtime/tui.ts:77-146`); backend `self.get` (as a resource dependency) or + `ctx.get` inside calls; code outside Chord calls (Durable tools) uses `host.root`. + +### 1.4 Principals + +`src/runtime/principals.ts`: `P2Principal` is `owner` (`access: "local" | "relay" | "device"`, `device` with +`"device"`), `guest` (one session, `mode: "view" | "full"`, grant `revision`), `worker` (session + generation, stamped +by the server route), `server` (the server's own requests on a worker) and `host` (a host's own code). Plugins import +`defineFacet`, `Ctx`, `Host` and friends from `src/runtime/chord.ts`, Chord's API typed by `P2Principal`, so factories +narrow on `principal.kind`. Helpers: `isOwner`, `isGuest`, `isWorker`, `canView`, `canSend`, `authorIdOf`, `authorOf`. +`mayReach` (`src/runtime/server.ts:158`) adds `SessionAccess`. + +Who gets which facet code: web and TUI facets are the same for owners and guests (both receive the selected facets' +bundles; `src/hosts/web.ts`, `src/hosts/server.ts`). UI gating (`useOwner()`, TUI `when: "owner"`) is cosmetic; +contracts and factories are the enforcement. + +### 1.5 Hot reload per facet + +| Facet | Trigger | What happens | +| --- | --- | --- | +| Server | explicit only: Reload button, `npm run reload`, `reload_plugins` (dev mode; `docs/p2-notes.md` "Dev mode") | Chord cluster transaction with every running worker: compatible changes keep state and resources; registrations swap atomically; contract/state changes need versions or migrations; a `registry_conflict` or failure keeps the old code. `src/runtime`, `src/hosts` need a restart. No reload without `P2_DEV=1`. | +| Worker | same transaction | as server; new workers spawn with the committed list. A Durable phase keeps the extension snapshot it started with (`src/runtime/durable.ts:35-41`); `ExtensionSwitches` changes apply to the next phase. | +| Web | backend reload builds a new content-named bundle; a visible page polls `/__p2_web` every 5 seconds | Facets are installed once per page (`startWeb`, `src/hosts/web/main.tsx`); any manifest-version change reloads the page. Foregrounding checks immediately; at most two previous generations remain available for up to 60 seconds. | +| TUI | backend reload builds a new bundle; the launcher polls the hash every second; `/reload` | Whole worker swap carrying bounded UI state (`docs/p2-notes.md` "TUI"). Within one worker: a `TuiCommands` change closes overlays (`src/runtime/tui.ts:461-465`); a changed `TuiSlots` value disposes and recreates that surface (`:471-485`). | + +## 2. Server facets + +### 2.1 Objects the server host serves + +| Contract | Kind | What | Pointer | +| --- | --- | --- | --- | +| `Server` | local, lifetime "host" | `info` (dataDir, fake, allowedHosts, port), `revokeOwners`, `revokeGuests`, test `resourceOptions` | `src/runtime/server.ts:44` | +| `Hosts` | local, lifetime "host" | `workers` (per-session running/generation/pid/error), `stop` | `:185` | +| `HostLifecycle` | local | `manager` (opaque `SessionHosts`: `pause`, `relocate`, `retry`, `stop`) | `:194`, `:228-493` | +| `Access` | remote | caller is owner, or guest with its grant | `:535`, `src/runtime/access.ts:14` | +| `TuiPreferences` | remote | server defaults for TUI keys/theme/settings (`tui.json`) | `src/runtime/tui-preferences.ts:52` | + +### 2.2 Registries + +| Registry | Key → value | Consulted | Used by | Notes | +| --- | --- | --- | --- | --- | +| `Routes` | `"METHOD /path"` or `"METHOD /prefix/*"` → `(req, res, {principal, origin}, ctx)` | after host paths, longest prefix wins (`src/hosts/server.ts:459`, `src/runtime/server.ts:124`) | files, artifacts, workspace, sharing, devices | Handler authorizes itself; `principal` may be `undefined` (stranger), owner, guest. | +| `Upgrades` | exact path → `(req, socket, head, principal, ctx)` | WebSocket upgrades other than `/chord` (`src/hosts/server.ts`) | terminals `/terminal` (`plugins/terminals/server.ts:22`) | Only authenticated principals reach it. | +| `Authenticators` | name → `(req, origin, ctx) => principal` | every request; first owner wins, else first principal (`src/hosts/server.ts:343-355`) | devices (owner cookie), sharing (guest cookie) | | +| `RequestResolvers` | name → `(req, ctx) => {principal, origin}` | before Host checks (`src/hosts/server.ts:357`) | radius, devices | Trusted transports only. | +| `SessionAccess` | name → `(principal, session, ctx) => boolean` | `mayReach`: worker routes, files (`src/runtime/server.ts:158-165`) | sharing (`plugins/sharing/server.ts:332`) | Any yes admits a non-owner. | +| `WorkerInitValues` | key → `(session, ctx) => value` | before each worker spawn; a throw refuses it (`src/runtime/server.ts:297-303`) | sessions (`plugins/sessions/server.ts:103`, key `session`) | Worker reads `WorkerProcess.init.values[key]`. | +| `SessionArchived` | name → `(session, ctx) => Promise` | awaited before `archive` returns (`plugins/sessions/server.ts:69`) | ports (`plugins/ports/server.ts:42`) | Defined by the sessions plugin. No restore/delete/create counterparts. | + +```ts +// plugins/sharing/server.ts:326 +self.register(Authenticators, { + guest: async (request, origin, ctx) => { + const cookie = cookieValue(request, cookieName(origin.port)); + return cookie === undefined ? undefined : ((await shares.authenticate({ cookie }, ctx)) ?? undefined); + }, +}); +``` + +Reserved host endpoints are `/health`, `/__p2_dev`, `/__p2_tui[/.mjs]` and the `/chord` WebSocket upgrade +(`src/hosts/server.ts`). Plugin `Routes` run before the web host, so they can shadow app pages, `/__p2_web[/…]` and +`public/` files. + +### 2.3 Dev, logs, notices + +- **Logs:** `processLogs` (`src/runtime/logs.ts:196`) is a process global: `append(source, level, text)`, + `notice(source, message)` (error-level owner notice, one per source, `:157-164`), `redactor.addSecret/addDirectory` + (models, login, radius use it), `persistLogSource`/`capturePublishedLogs` for detached processes (terminals). + Worker stderr lines starting `p2 worker:` become notices (`src/runtime/server.ts:335-343`). +- **Notices:** surfaced by the logs plugin's owner-only `LogNotices` (`plugins/logs/contract.ts:24`), rendered by the + core web lane (`src/web/OwnerNotices.tsx:30-31`) and the logs TUI facet. There is no tone, action or dismiss API for + plugins: only `processLogs.notice` (errors). +- **Dev:** no registry. The dev channel is host code (`src/hosts/dev-channel.ts`); the dev-tools worker plugin + imports it directly (`plugins/dev-tools/worker.ts:5`). There is no "after reload" hook: resources with a `version` and + Chord's per-cell migrations carry state across reloads. + +## 3. Worker facets + +### 3.1 Durable extensions (`Extensions`) + +`Extensions` (`src/runtime/worker.ts:57`): key = extension name, value = a Durable `Extension` +(`packages/durable/src/harness/types.ts:365-374`). Every conversation selects every installed extension unless its +`pi.agent.extensions` says otherwise (`packages/durable/src/harness/types.ts:530-532`); the Harness reads the registry +through `durableRegistry` (`src/runtime/durable.ts:41`). + +| Part | Field | What | p2 users | +| --- | --- | --- | --- | +| Tools | `tools` (`defineTool`, `types.ts:293`) | `parameters`, `execute(args, api, ctx)` returning `output`/`structuredOutput`/`details`; `replay: "safe" \| "unsafe"` (rerun after a crash, `:298`); `callers: ["model" \| "tools"]` (`:317`); `structuredOutputSchema` (`:311`); `deferred` (`:322`); `executionMode`, `prepareArguments`, `outputLimits`; `api.executeTool` for nested calls (`:281`) | coding-tools, artifacts, send-file, ports, conversations, sessions, subagents, dev-tools, codemode, powershell; `agent` is `callers: ["tools"]`, replay-safe with a schema (`plugins/agent/tool.ts:53-57`) | +| Prompt | `sections` (`section(key, render, {tag})`, `types.ts:349`) | rendered before each request, in extension order | prompt (`plugins/prompt/worker.ts:185-191`), coding-tools, ports, conversations, dev-tools | +| Hooks | `hooks: [hook(Task, handlers)]` | `GenerationTask`: `beforeRequest`, `afterResponse`, `onYield`, `afterTools` (`types.ts:757`); `ToolTask`: `beforeTool`, `afterTool` (`:778`); `CompactionTask`: `beforeCompact` (`:795`); handlers get `HookApi` (doc reads, memos, models, `:745`) | none yet | +| Wraps | `wraps: [wrapTool(tool, fn) \| wrapSection(section, fn)]` (`types.ts:360`) | replace another extension's tool or section where selected | none yet | +| Tasks | `tasks: [defineTask(…)]` | durable background work, resolved by name | subagents (`Anchor`, `Reporter`, `plugins/subagents/tools.ts:72,118,473`) | + +```ts +// plugins/ports/worker.ts: the tool calls the server's Ports object from outside Chord through host.root +const extension = createPortsExtension( + (query, origin, context) => + host.root((ctx) => ctx.get(Ports, {}).query({ conversation: 1, origin, query }, ctx), { + signal: context.abortSignal, + }), + host.name.slice("session:".length), +); +contribute(host, "p2.port-tools", (self) => self.register(Extensions, { [extension.name]: extension })); +``` + +Who: tools run for whoever's input started the turn. Durable knows no principal; tools that need one derive it from +the calling assistant entry with `originOf` (ports, dev-tools). Reload: a running phase keeps its snapshot; tool code +that must survive reload reads live resources through getters (`plugins/codemode/worker.ts:37-43`). + +### 3.2 Worker registries and locals + +| Hook | Kind | What | Consulted | Used by | +| --- | --- | --- | --- | --- | +| `ExtensionSwitches` | registry: extension name → `Readable` | `false` leaves that extension out of new snapshots | `src/runtime/durable.ts:31,43-93` | codemode (`codemode` setting), powershell (Windows), dev-tools (dev checkout) | +| `InputGates` | registry: name → `(ctx) => Promise` | awaited before any input is submitted | `plugins/chat/worker.ts:320,380,404`, `plugins/conversations/worker.ts:132` | prompt (first AGENTS.md load) | +| `SendRouters` | registry (chat): name → `(RoutedSend, ctx) => {submissionId} \| undefined` | `Chat.send` asks each; untaken sends become `p2.input` | `plugins/chat/worker.ts:571` | chat-mode (`plugins/chat-mode/worker.ts:134`) | +| `ReadOnlyConversations` | registry: name → `(conversation, ctx) => reason?` | refuses `Chat.send`/`dequeue`/`configure` | `plugins/chat/worker.ts:514` | subagents (one-shot agents) | +| `ConversationConfigured` | registry: name → `(conversation, ctx)` | after a browser/TUI `configure` | `plugins/chat/worker.ts:482` | subagents | +| `HarnessConfig` | registry, key `"default"` only | models + Durable settings | `src/runtime/worker.ts:332-350,410` | models | +| `HarnessEnvironments` | registry, key `"remote"` only | opens a remote machine (`at(cwd)` environments, one connection) per environment name | `src/runtime/worker.ts` | envs | +| `ConversationLocations` | local, one provider | conversations following the session, own locations (`place`, `own`) | `src/runtime/worker.ts` | subagents (`plugins/subagents/worker.ts`) | +| `Harness` | local, lifetime "host" | `harness` (native Durable Harness), `environments` (`session`, `locate(id)`, `at(location)`) | `src/runtime/worker.ts` | most worker plugins | +| `WorkerProcess` | local, lifetime "host" | `init` (dataDir, generation, `values` from `WorkerInitValues`), `liveness` (`add("viewers" \| "activeCalls")`, `stopping`) | `src/runtime/worker.ts:165` | chat and workspace (viewers), codemode (`stopping`) | +| `Conversation` | local (chat) | `submit` (input with own `entry` kind), `write` (display-only entry), `stop`, `history`, `compact`, `reset`, `runShell`… | `plugins/chat/contract.ts:140-189` | chat-mode, conversations | + +```ts +// plugins/chat-mode/worker.ts:134 (shortened) +self.register(SendRouters, { + "chat-mode": async ({ conversation, author, text, requestId }, ctx) => { + if (!(await enabled(conversation, ctx))) return undefined; // leave it to the next router / p2.input + const chat = ctx.get(Conversation, { conversation }); + return chat.write({ kind: CHAT_ENTRY, data: { author, text, attachments: [] }, requestId, author }, ctx); + }, +}); +``` + +### 3.3 Entries, documents, environments + +- **Entry kinds:** `Conversation.submit({ …, entry: { kind, data } })` places an input under a plugin kind + (`plugins/chat/contract.ts:127-128`); `write` appends display-only entries (`:150-158`). Data that is a person's + message (`MessageDataSchema`, `src/runtime/input.ts:144`) is shown by both UIs; other model-less kinds are hidden on + the web (`src/web/transcript.ts:506-512`) and need a `TuiEntryRenderers` entry in the TUI. +- **Documents:** `defineDoc` from `@earendil-works/pi-durable`; write in `harness.commit(tx => tx.doc(Doc, id))`, + read with `harness.snapshot(Doc, id)` or in tools/hooks via their API. Conversation docs reach clients in + `Chat.view.docs`. Examples: `p2.subagents`, `p2.agent-run`, `p2.artifacts`, chat-mode's `ChatModeDoc` + (`plugins/chat-mode/worker.ts:25`). Change signals: `CommitSignal` (`src/runtime/commits.ts:4`), `watchSource` + (`src/runtime/durable.ts:113`). +- **Environments:** tools use `api.env`; plugins use `Harness.environment` / `environments.at(cwd)`. Only one remote + provider (`HarnessEnvironments["remote"]`). + +## 4. Web facets + +### 4.1 Slots + +`Slots` (`src/runtime/web.tsx:49`): key → React component. `` renders one (or its children/fallback), +`` renders every `prefix:*` key. Every rendering site today: + +| Slot | Rendered at | Kind | Filled by | +| --- | --- | --- | --- | +| `app` | `src/hosts/web/main.tsx` (`startWeb`) | single | web-shell | +| `sidebar` | `src/web/AppLayout.tsx:21` | single | sessions | +| `main` | `src/web/AppLayout.tsx:60` | single | chat | +| `workspace-panel` | `src/web/AppLayout.tsx:62` | single | workspace | +| `artifact-actions`, `terminal-actions`, `workspace-actions`, `session-options`, `session-actions` | chat top bar, `plugins/chat/web.tsx:327-331` | single each | artifacts, terminals, workspace, sessions, sharing | +| `composer:*` | `plugins/chat/web.tsx:925` (`SlotList`) | list | chat-mode | +| `chat-bottom` | `plugins/chat/web.tsx:934` | single | terminals | +| `chat-content` | `plugins/chat/web.tsx:1133` | wrapper (gets `children`) | artifacts (viewer contexts) | +| `chat-attribution` | `plugins/chat/web.tsx:1134` | wrapper | conversations | +| `sidebar-foot:providers\|envs\|devices\|ports\|logs` | `plugins/sessions/web.tsx:29-37` | five fixed `Slot`s, not a list | login, envs, devices, ports, logs | +| `envs-actions:*` | `plugins/envs/EnvsPanel.tsx:524` (`SlotList`) | list | ports | +| `workspace-terminal` | `src/web/Workspace.tsx:730` | single | terminals | +| `chat-parent` | **nowhere** | — | subagents registers it (`plugins/subagents/web.tsx:22`) | + +### 4.2 Sheets, navigation, hooks + +| API | Pointer | Use | +| --- | --- | --- | +| `openLayer({kind: "sheet" \| "viewer", name, locator?}, {fromDrawer?})`, `closeLayers()` | `src/runtime/web.tsx:296,302` | push a named modal; Back and reload restore it | +| `useSheet(name, kind?)` | `:316` | the top layer if it is yours, with its locator | +| `` | `:91` | shared modal (portal, Escape, scroll memory) | +| `useNavigation()`, `useRoute()`, `navigate()` | `:275-293`, `src/web/navigation.ts` | push/replace/close; never browser history | +| `useObject`, `useReadable`, `useStatus` | `:139,183,195` | hold objects, read state | +| `useOwner()` | `:325` | owner check (UI only) | +| `useConnected()`, `useShareEnded()` | `:225,247` | connection, ended guest share | +| `useHost()`, `useRegistry()` | `:42,52` | raw host and registry tables | + +Pattern: the plugin's slot component renders its button and, when `useSheet(name)` matches, its sheet +(`plugins/envs/web.tsx:6-21`). + +### 4.3 Composer, transcript, notices, theme + +- **Composer:** `ComposerExpanders` (`src/runtime/composer.ts:7`): `(host, session, text) => text?`, asked in key order + for text starting with `/` (`plugins/chat/web.tsx:621`); prompt registers one (`plugins/prompt/composer.ts:10`). + Chips: `composer:*` slots. Shared state: `useComposerState()` (`src/web/composer-state.tsx:123`: draft, files, + `addFiles`, `replyTo`, `canReply`/`setCanReply`, `pinChat`). `@`/Tab completion is core (`src/web/file-completion.tsx`). + No web commands, slash commands or key bindings. +- **Transcript:** `TranscriptEntries` (`plugins/chat/contract.ts:270`): entry kind → pure `(data) => {replyTo?, note?}`, + adds to a person's message (chat-mode, shared with the TUI). Tool cards are a fixed `switch (tool.name)` in + `src/web/Chat.tsx:303-511` (bash, codemode, read, write, subagent, artifact, ports, send_file, edit, generic). React + contexts the chat plugin provides to cards (`src/web/Chat.tsx:65-77`: `OpenConversation`, `OneShotAgents`, + `OpenPeerConversation`, `WorkingDirectory`, `OpenWorkspaceFile`, `SessionId`, `ReplyTo`) are not a plugin API. +- **Owner notices:** one core lane (`src/web/OwnerNotices.tsx`) fed by `LogNotices` and the dev status; plugins can only + publish server errors (2.3). +- **Theme:** `src/web/theme.ts` (system/light/dark, `setTheme`/`subscribeTheme`) and CSS in `src/web/style.css`; plugins + import their own CSS (`plugins/chat-mode/style.css`). No theme contribution. + +## 5. TUI facets + +### 5.1 Slots and surfaces + +`TuiSlots` (`src/runtime/tui.ts:50`): key → `(runtime) => TuiSurface` (`:32-45`: `render`, `dispose`, `focus`, +`contexts()`, `layout()`, `saveState()`). `tuiFacet(name, factory, commands)` (`:52`) registers one slot plus commands. +Every factory is instantiated when registered (`:480`), rendered or not. + +| Key | Rendered | Filled by | +| --- | --- | --- | +| a name without `:` (except `header`, `footer`, `status`) | as a screen; also a mutually exclusive `when` term (`:397-401`); `ctx.navigate(name)` | `sessions`, `chat` | +| `header` | replaces the built-in header (`:254`) | — | +| `footer` | replaces the built-in footer (`:497`) | status | +| `footer:*` | first one only, inside status's footer (`plugins/status/tui.ts:153`) | workspace `footer:branch` | +| `composer:*` | chips under the editor, key order (`plugins/chat/tui.ts:78`) | chat-mode | +| anything else with `:` (`service:*`, `widget-above:*`) | never rendered: used as per-runtime background holders | logs, models, prompt, sharing | + +### 5.2 Commands, keys, slash + +`TuiCommands` (`src/runtime/tui-commands.ts:29-49`): key = command id (pi ids such as `app.model.select` where the +meaning matches, else `.`; `tui.*` and launcher ids are refused, `:260`). Value: `title`, `category`, +`keys`, `when`, `slash: {name, argumentHint, complete, tabOnly}`, `palette`, `run(ctx, args)`. + +- `when` grammar (`:65-113`): `&&`, `||`, `!`, parentheses over terms: screen names, `owner`/`guest`, `connected`, + `overlay`, `overlay:`, and the active surface's `contexts()` (`src/runtime/tui.ts:403-411`). More terms win. +- Key conflicts, launcher-owned keys, slash collisions are reported in `/hotkeys`, never thrown (`:248-330`). +- Visitors rebind ids in pi's `keybindings.json` or `prefs.json`; server defaults in `tui.json`. + +```ts +// plugins/artifacts/tui.ts:192 +"p2.artifacts": { title: "Artifacts", category: "Session", when: "chat && !overlay", + slash: { name: "artifacts" }, run: (ctx) => openArtifacts(ctx) }, +``` + +`TuiSlashSources` (`:57-62`): slash names that are not commands (`entries(ctx)`, `expand(ctx, text)`); commands win. +Prompt registers templates and skills (`plugins/prompt/tui.ts:87`). + +### 5.3 Renderers + +| Registry | Key → value | Order | Users | +| --- | --- | --- | --- | +| `TuiToolRenderers` | tool name → pi `ToolRenderers` (`renderCall`, `renderResult`, `renderShell`; `src/tui/tools/types.ts:15-62`) | overrides built-ins | artifacts, ports, send-file, subagents | +| `TuiToolRendererResolvers` | `:` → `(toolName, next) => renderers` | key order, then the registry, then built-ins (`src/tui/tools/index.ts:20-28`) | none yet | +| `TuiEntryRenderers` | entry kind → `(entry, ctx) => Component \| undefined` (replaces the display; `undefined` hides) | — | core `p2.shell` (`src/tui/core-commands.ts:290`) | +| `TuiMarkdownTransformers` | `:` → pi `MarkdownTransformer` | key order | core `00:mermaid` | +| `TranscriptEntries` | shared with web (4.3) | — | chat-mode | + +All in `src/runtime/tui-renderers.ts:16-45`; consumed by the chat surface (`plugins/chat/tui.ts:226-248`). + +### 5.4 Sessions, overlays, status, footer + +- `TuiSessionSources` (`src/runtime/tui-sessions.ts:22`): `children(runtime, onChange)` and `search(runtime, query, + archived)` for the session picker (subagents, search). `TuiSessionPlaces` (`:34`): `where(ctx)`, `moveTo(ctx, env)` + (sessions; used by envs). +- Overlays: `ctx.overlay(id, component, options)` (`src/runtime/tui-context.ts:112`); overlay-scoped commands run + first; closed on navigation, reload and command-table changes. +- Status lines: `ctx.notify(message, kind)` (`:107`), last three, cleared on navigation (`src/runtime/tui.ts:258-264`). +- Footer: replace it (`footer`) or add one `footer:*` line part; no ordered footer items. +- Other `TuiContext` members (`src/runtime/tui-context.ts:91-126`): `host`, `theme`, `keys`, `hints`, `settings`, + `role`, `identity`, `navigate`, `view` (expanded tools, thinking), `commands` (run, runSlash, palette, list, report), + `surface(name)`, `exit`, `reload`, `prefs` (theme names, preview, `save` via grant). Plugins also reach into the chat + surface: `ctx.surface("chat") instanceof ChatSurface` (`plugins/send-file/tui.ts:93`, `plugins/chat-mode/tui.ts:70`). +- Theme: per runtime, the visitor's (`ctx.theme.current`); plugins cannot add themes. + +### 5.5 Launcher grants + +`ctx.grant(action, payload)` asks the visitor's launcher (`src/hosts/tui-actions.ts:376-700`). Only `C` and `gesture` +actions can be requested; `K` actions fire on the visitor's own key and may ask the worker for data through +`runtime.launcherAnswers` (`src/runtime/tui.ts:197`, `plugins/chat/tui-commands.ts:139`). Anything else is denied +(`src/hosts/tui-client.ts:226-259`). + +| Action | Kind | Payload | Owner only | +| --- | --- | --- | --- | +| `attach` | C | `{path?}` → uploaded file | no | +| `copy` | C | `{text, label?}` | no | +| `openUrl` | C | `{kind: "external", url}` (https) or `{kind: "origin", path}` | no | +| `download` | C | `{kind: "file" \| "artifact", session, …}` | no | +| `savePrefs` | C | `{patch}` for `prefs.json` | no | +| `devCommand` | C | `{action: "reload"}` | yes | +| `signOut` | C | `{all}` | yes | +| `copy.selection` | gesture | selection text after the launcher saw the mouse release | no | +| `attach.key`, `paste.key`, `copy.key`, `editor.external`, `suspend` | K | — (visitor's key) | no | + +## 6. Inconsistencies + +1. **Tool cards.** TUI: `TuiToolRenderers` + ordered resolvers, pi's contract. Web: a hard-coded switch in core + (`src/web/Chat.tsx:303-511`) that knows plugin tools (artifact, ports, send_file, subagent) and imports a plugin + contract (`src/web/Chat.tsx:12`). A new plugin's tool gets the generic JSON card on the web. +2. **Entry display.** TUI `TuiEntryRenderers` replaces an entry's display; web has only `TranscriptEntries` (adds a + reply quote and a note to a person's message) and silently drops other model-less kinds + (`src/web/transcript.ts:506-512`). Core reads plugin documents (`view.docs["p2.artifacts"]`, + `src/web/transcript.ts:371`). +3. **Commands.** TUI has one registry for keys, palette, slash and help with a `when` grammar. Web has none: no + slash commands (only `ComposerExpanders` for `/template`), no palette, no shortcuts. The same action (compact, + model, artifacts, share) is a `TuiCommand` in the TUI and a hand-placed button in the web. +4. **Same job, two registries.** Prompt expansion is `ComposerExpanders` (web, `(host, session, text)`) and + `TuiSlashSources` (TUI, `(ctx, text)` plus `entries`); the prompt plugin registers both. Composer chips are `Slots` + `composer:*` (registration order) and `TuiSlots` `composer:*` (key order). +5. **Naming.** TUI registries are `Tui*`; web ones are unprefixed (`Slots`, `ComposerExpanders`); `TranscriptEntries` + lives in the chat plugin but is used by both UIs; worker `SendRouters` lives in chat, `ReadOnlyConversations` + in the runtime, `SessionArchived` in sessions. Ordered registries are marked only by convention in key strings. +6. **Slot lists that are not lists.** `sidebar-foot:*` is five fixed names (`plugins/sessions/web.tsx:29-37`): a + new plugin's `sidebar-foot:foo` never renders. `footer:*` in the TUI renders only the first entry. `chat-parent` + is registered and rendered nowhere. Wrapper slots (`chat-content`, `chat-attribution`) admit one plugin each. +7. **Slots as lifecycle.** TUI plugins register invisible `service:*` / `widget-above:*` slots to get a per-runtime + object, then find it with `ctx.surface(name)` (`plugins/prompt/tui.ts:74-80`, `plugins/models/tui.ts:116-118`). + The designed `widget-above`/`widget-below` rendering (`docs/p2-tui-port.md` §7.2) does not exist. +8. **The chat surface as API.** TUI plugins import `ChatSurface` and call its methods (`toolViews`, `setCanReply`); + web plugins use `useComposerState()` from `src/web`. Neither is a declared contract. +9. **Core knows plugins.** `Route` has `workspace`/`terminal` + fields (`src/web/navigation.ts:20-23`); `OwnerNotices` imports the logs contract (`src/web/OwnerNotices.tsx:3`); + the server reports `hasLogs: plugins.includes("logs")` (`src/hosts/server.ts:424`); `settings.json` has a closed + schema with plugin keys such as `codemode` (`src/runtime/settings.ts:63-68`); `src/web/Workspace.tsx` is the + workspace plugin's UI in core. +10. **Built-ins use internals.** dev-tools imports `src/hosts/dev-channel.ts`; devices imports `src/hosts/principals.ts`; + sessions and status drive `HostLifecycle.manager` (opaque `SessionHosts`); models, login and radius call the process-global + `processLogs.redactor`. Single-key registries (`HarnessConfig["default"]`, `HarnessEnvironments["remote"]`) and + the single-provider `ConversationLocations` mean only one plugin can ever provide models, remote environments or + conversation locations. +11. **Notices.** Owner notices are server errors only (`processLogs.notice`), shown by core web code and by the logs + TUI facet; TUI `ctx.notify` is local; the web has no local notification API for plugins. +12. **Reload.** Server/worker reload in place; TUI swaps the worker; web facets take effect through a page reload. + Registries behave the same, their consumers do not. + +## 7. What is lacking (compared with pi 1.0 extensions) + +Reference: `packages/coding-agent/docs/extensions.md`. + +| pi 1.0 | p2 today | Gap | +| --- | --- | --- | +| `pi.on()` lifecycle: `session_start`/`shutdown`, `before_agent_start`, `context`, `message_end`, `tool_call`/`tool_result`, `turn_end`, `agent_before_settle`, `session_before_compact`, `user_bash`, `input`, `provider_stream_event`, `cache_warming_decision` | Chord resources for start/shutdown; Durable hooks cover `context` (`beforeRequest`), `message_end` (notify-only `afterResponse`), `tool_call`/`tool_result`, `turn_end` (`afterTools`), settle (`onYield`), compaction; `SendRouters` covers `Chat.send` input only | No input hook for peer/subagent/agent inputs; no `user_bash` (shell is hard-coded in chat); no stream events; no whole-prompt override (sections and `wrapSection` instead). No plugin uses hooks yet, so they are untested in p2. | +| `registerTool` with `exposure`, `namespace`, `annotations`, `prepareLoadout`, `reachesTools` | `callers`, `deferred`, replay policy | No annotations (no approval policy), no namespaces; `tool_search` not installed. | +| `registerCommand`, `registerShortcut` | TUI only | No web commands or shortcuts. | +| `registerFlag` | none | Plugins get configuration only via env vars or their own files; `settings.json` is closed. | +| `registerProvider`, `registerVirtualModel` | `models.json` and `HarnessConfig["default"]` owned by the models plugin | No provider or virtual-model contribution. | +| `registerMcpServer` | none (open item in `docs/handoff.md`) | No MCP. | +| `sendUserMessage`, `sendMessage`, `appendEntry` | `Conversation.submit` / `write` (worker), `Chat.send` (clients) | Equivalent. | +| `setActiveTools`, model and thinking control | `configure` on Durable `pi.agent`; `ExtensionSwitches` | Equivalent, per conversation. | +| `registerToolRenderer`, message renderers | TUI only | Web has neither (6.1, 6.2). | +| `ctx.ui` dialogs (`confirm`, `select`, `input`), widgets, status, title | TUI overlays and status lines inside the TUI; nothing a tool can ask | A tool cannot ask the person (ports asks the model to obtain confirmation in chat). | +| `pi.events` between extensions | Chord contracts and registries | Stronger in p2. | +| packages with skills, prompts, themes | built-in plugins only | Designed in `plugin-loading.md`. | +| themes | TUI: visitor's own; web: fixed CSS | No plugin themes. | + +## 8. Proposals + +Ordered by value; each keeps one name and one shape across surfaces. + +1. **Tool and entry renderers on the web.** Add `WebToolRenderers` (tool name → `{ card(tool, ctx) }`) and + `WebToolRendererResolvers` (`:`, `next()`), mirroring the TUI pair, and `WebEntryRenderers` + (entry kind → component). Move each case of `src/web/Chat.tsx:303-511` into its plugin (coding-tools: bash, read, + write, edit; codemode; subagents; artifacts; ports; send-file). Pass today's React contexts as one documented + `CardContext`. Core then reads no plugin documents or contracts. +2. **One command model.** Move the command shape (`id`, `title`, `category`, `when`, `slash`, `keys`) to a + surface-neutral module with `run(ctx)` where `ctx` has the shared part (host, identity, role, navigate, notify), + and keep `TuiCommands` plus a new `WebCommands` that share ids (`app.session.compact` on both). The web gets slash + commands in the composer, a palette and shortcuts from the same entries. +3. **One slash source.** Replace `ComposerExpanders` and `TuiSlashSources` with `SlashSources` + (`entries(identity)`, `expand(identity, text)`) used by both composers. +4. **Consistent naming and order.** Prefix per surface (`WebSlots`, `WebComposerExpanders` → or shared names when + shared); give registries an explicit `ordered: true` option (or an `order` field in values) instead of key + conventions; make `SlotList` key-ordered like the TUI. Move shared registries (`TranscriptEntries`) to + `src/runtime` or a neutral contract module. +5. **Real lists.** `sidebar-foot:*` and TUI `footer:*` as ordered lists; delete or render `chat-parent`; make + wrapper slots chains (`chat-content:*`). +6. **A TUI service registry.** `TuiServices` (`name → (runtime) => { dispose() }`) for per-runtime holders, and + render `widget-above:*`/`widget-below:*` as designed; replace `ctx.surface("chat") instanceof ChatSurface` with a + `ctx.chat` interface (editor text, tool views, replies) and expose the same on the web instead of + `useComposerState()` internals. +7. **Owner notices as a contract.** A server `Notices` registry/object (tone, message, actions as command ids, + dismiss) with web lane and TUI status lines as two renderers; `processLogs.notice` becomes one producer. +8. **Plugin settings.** `settings.json` `plugins.` validated by a schema each plugin registers + (`SettingsSchemas`), hot-reloaded like today; moves `codemode` out of core. This also covers pi's flags. +9. **Multi-provider registries.** `HarnessConfig` → `ModelProviders` (several plugins add providers, merged into + `Models`); `HarnessEnvironments` keyed by environment kind; `ConversationLocations` as a registry. Unblocks + `registerProvider`-like plugins and MCP. +10. **Hooks that p2 owns.** An input hook for every submission path (Chat, peer, subagent, agent), a shell hook for + `Chat.shell`, and a worker-side `ToolApprovals` question document with web and TUI renderers, so a tool can ask a + person (pi's `ctx.ui.confirm`) and annotations can drive it. +11. **Core free of plugin names.** Web facets loaded from the plugin graph like the other surfaces (plugin-loading.md); + `Route` extension data per plugin (`route.plugins[name]`); no `plugins.includes(...)` in hosts; dev-tools through a + `DevControl` contract instead of `src/hosts/dev-channel.ts`. diff --git a/apps/p2/docs/plugin-loading.md b/apps/p2/docs/plugin-loading.md new file mode 100644 index 000000000..be9e993de --- /dev/null +++ b/apps/p2/docs/plugin-loading.md @@ -0,0 +1,711 @@ +# Plugin loading + +Status: **design, not implemented** (revision 4), except for the built-in parts of 4.1 and 4.2 (listing, runtime +resolution), which are. Revision 4 aligns the document with `target-design.md`: a plugin reload swaps code on each host +without waiting for running work (documents migrate, resources are replaced in place, section 15 there), and the +cluster is a rollout of revisions, not a transaction. Loader model D (section 14) stays: one base bundle plus one script +per facet, evaluated without a module map, so old generations are reclaimed by the GC. p2 is experimental; nothing here +is a stable public API. + +**Chord v2 only.** This document predates p2's move to Chord v2; Chord v1 has since been deleted. Where it says +"today" about `nodeLoader` and per-surface loading, it describes p2 on Chord v1. p2 now builds and loads its server +and worker facets with Chord's plugins component (`createPlugins`, `src/hosts/plugins.ts`; Chord's +`docs/plugin-authoring.md`), plugins import p2's API (`@p2/app/v1` and its per-kind modules) instead of Chord +directly, and the web and TUI bundles load their facets with `host.load`. Global, package and project plugins +(sections 3 to 13) remain a design. + +Today p2 loads one plugin root, `apps/p2/plugins/`, differently per surface: server and worker facets through +Chord's `nodeLoader` (dynamic `import()`, a new module copy per reload that is never evicted), TUI facets through a +server-built esbuild bundle, web facets from a hard-coded list in `src/hosts/web/main.tsx` served by Vite. An +uncommitted prototype (`src/hosts/worker-bundle.ts`) measures worker bundle variants (14.1). This document designs +one mechanism for built-in plugins, the owner's own plugins, plugins installed from npm or git, and plugins that come +with a session's working directory, on every surface. + +## 1. Goals and non-goals + +Goals: + +- **One pipeline** for built-in, global, package and project plugins; built-ins are not special. +- **Every surface from the same plugin set:** server, session worker, web, TUI. +- **Global packages from npm and git**, like pi's packages (`packages/coding-agent/docs/packages.md`). +- **Project plugins** from a session's working directory, local or remote, without leaking into other sessions. +- **Nothing from a project runs before the owner approves it, and only the approved bytes run.** +- **No leaks across scope changes:** a web page, TUI worker or session worker that loaded a plugin set is thrown + away as a whole, never partially unloaded. +- **In-place reload everywhere, without leaks:** a changed facet reloads through Chord (documents and resources stay, + running calls finish with their old code) and its old generation is reclaimed by the GC once nothing references it + (14). A full reload (host restart, page load, TUI worker swap) remains for base changes, scope switches and stuck + generations. +- **Publishable later:** p2 as an npm package loads external plugins the same way. + +Non-goals (v1): + +- Isolation between plugins in one process, or of web code from the page (that needs iframes; 11). +- Project packages (npm dependencies of project plugins); project plugins import only host packages and plugins. +- Project web or TUI facets for guests (base scope only; opt-in per share later). +- Native addons in packages. + +## 2. Terms + +- **Plugin:** a directory with facets (`server`, `worker`, `web`, `tui` modules) and the modules they import, and/or + resource directories (`skills/`, `prompts/`, `themes/`). Its name is the directory name, `[a-z0-9-]+`. +- **Facet:** the part of a plugin that runs on one surface. +- **Root:** a directory whose subdirectories are plugins. +- **Package:** an npm or git source, or a local directory, providing one or more plugins. +- **Scope:** the plugin set one surface instance loads. **Base scope:** built-in, global and global-package plugins. + **Project scope:** the base scope plus the approved project plugins of one directory. +- **Scope key:** `(env id, canonical session cwd)`. **Revision:** the digest of an immutable project snapshot (6.2). +- **Host packages:** packages the running p2 supplies to every plugin as single instances: p2's API + (`@p2/app/v1`, re-exporting the part of Chord v2 plugins use, with per-kind modules), `@earendil-works/pi-ai`, `pi-codemode`, `pi-durable`, `pi-env`, `pi-tui`, `pi-schema` (all `@earendil-works/`), + `react`, `react-dom` (with subpaths such as `react/jsx-runtime`, `react-dom/client`), and p2 itself + (`@earendil-works/p2/runtime/*`, `@earendil-works/p2/plugins/*`). +- **Data directory:** `P2_DATA_DIR` (default `~/.pi/p2`), written `` below. + +## 3. Sources + +| Source | Where | Declared by | Facets | Gate | +| --- | --- | --- | --- | --- | +| Built-in | the app's `plugins/` | the app | all | none | +| Global | `/plugins//` | files present | all | the owner put them there | +| Global package | `/packages/…` (8) | `packages` in `/settings.json` | all | the owner installed it | +| Project | `/.pi/p2/plugins//` | files present | worker, web, tui | project trust (6) | + +- **Server facets never come from a project.** The server serves every session. A project plugin with a `server` + module is refused as a whole (owner notice); loading its other facets would leave it half-working. +- **Disabling:** `plugins.disabled` in `/settings.json` lists names not to load, built-ins included. A plugin + that holds a disabled plugin's contracts fails at activation with a notice naming it and the type (4.2). + `/.pi/p2/settings.json` (part of the project snapshot) may only disable **project** plugins; any other key is + ignored with a notice. +- **Project skills and prompts:** `/.pi/skills` and `/.pi/prompts`, which p2 loads today without asking + (`plugins/prompt/context-files.ts`), move under the same directory trust decision (pi parity). **Behavior + change:** they no longer load in untrusted directories. `AGENTS.md` files stay ungated, as in pi. + +## 4. Discovery, dependencies, names + +### 4.1 Discovery: a directory listing (implemented for built-ins) + +Discovery lists directories; it never parses, imports or runs plugin code (owner decision: p2 does not parse plugin +source at all). + +- A plugin is a directory under a root with facet files: `server`, `worker`, `web`, `tui` with `.ts`/`.tsx` (and, + for packages, `.js`/`.mjs`). `readdir` plus an existence check per facet kind is all the loader knows + (`listPlugins`, `src/hosts/plugins.ts`). Its name is the directory name; a facet module's plugin is the directory + it lives in (`pluginOfModule`). +- **Selection** is exactly the named plugins (default: all), loaded in stable alphabetical order. Nothing is added + for imports: a fixture or `plugins.disabled` that leaves out a provider makes its consumers fail loudly at + activation (4.2). +- **The server selects; workers scan nothing.** The server passes the selected plugins (name and worker facet + module) in each worker's init. The TUI bundle ships the TUI facet of every selected plugin. +- **Type ownership is recorded at runtime,** per host, from the `serve` calls of the facets that host actually + loaded (`PluginRuntime`). No process imports another kind's facets: a worker never imports server facets just + to learn names. Diagnostics name the consumer plugin and the missing type (`plugin login needs p2.model-config; + no loaded plugin serves p2.model-config on server`), not an unloaded provider, which nobody parsed. +- **Dev mode:** a save belongs to the plugin whose directory contains it. It marks that plugin dirty when it is a + backend facet file, a plugin set change (directory or backend facet added/removed), or a file some live server, + worker or TUI bundle actually loaded (the facet bundles' esbuild metafiles under D, 14.2); other saves (web-only + code) stay on Vite HMR until the web moves to D. Files outside plugin directories are runtime code, tracked by the + content of the modules each process loaded ("Server code changed"). +- **Dev loop:** a change to runtime code (the base bundle: Chord, Durable, other packages, `apps/p2/src/**`) restarts + the affected hosts (server, workers, page reload, TUI worker swap); a plugin change reloads that plugin in place on + every host that runs one of its facets (`npm run reload -- `, rollout in 7.5). + +### 4.2 Dependencies: runtime resolution + +A plugin refers to another plugin's modules in one of two ways: + +- **Relative imports** within its own root (`../chat/contract.ts`), as built-ins do. +- **`p2:plugins//`** across roots: a virtual specifier resolved **per scope** to wherever plugin `` + lives in that scope (built-in, global, package or snapshot), by the facet bundler's resolver (14.2). + `@earendil-works/p2/plugins//…` stays the name for built-in contracts (it resolves to the host's files). + +Under D each facet is its own bundle, so another plugin's file is **copied into the importing facet** (14.4, B2). +Cross-plugin imports are therefore limited to contracts and side-effect-free modules (`contract.ts`, `protocol.ts`, an +allowlist), and never used for identity (`instanceof`, module state, React contexts); a check enforces it (14.5). + +Imports are not dependencies. What a plugin needs is resolved by Chord at runtime, by contract name: + +- **Load order does not matter.** `host.load` installs every facet of a load before it activates any eager object + (Chord guide, "Startup readiness"), so an eager consumer may hold a type a later facet serves. Workers load with + `{ awaitEager: false }` and then await `host.ready()` once the server admitted their link (Chord request #11). + (Today facet modules of one load share one copy of each other's files because Chord's `nodeLoader` registers every + facet directory before importing; D drops that, see above.) +- **A missing provider fails at activation,** with Chord's `unavailable/unknown_type`, which p2 extends by the + consumer plugin's name before the hold activates (local types) or when the remote error comes back. +- **Removal safety is a runtime check.** A reload that removes a plugin first asks the server and every running + worker which plugins outside the batch still hold one of the removed plugin's types (live factory holds, + recorded per host). Any holder refuses the reload before any host prepares, naming them (`Cannot remove p2 + plugin dev-added: plugin dev-dependent still holds test.dev-added (server)`). While the reload prepares, a new + hold on those types, including the new code of a plugin reloaded in the same batch, refuses it as well. + +Project plugins may depend on base plugins and on project plugins of the same snapshot; base plugins never depend on +project plugins. Anything a project plugin imports must be its own file, a scope plugin or a host package; any other +specifier is a load error for that plugin. + +### 4.3 Names + +Chord binds services and registries **by name**: two contracts named `todo.store` in one host throw `already served`; +two Durable extensions defining one task name throw at registry construction (`durable/src/harness/registry.ts`), +and extensions are looked up by name. +Built-ins use `p2.` names. Rules for external plugins, checked at load (a violation refuses that plugin with a +notice): + +| Name | Required form | +| --- | --- | +| Chord contracts, registries | `.` | +| Durable extensions, task kinds | `.` | +| Agent tools | `_` (provider APIs allow only `[a-zA-Z0-9_-]{1,64}`; see A.1) | +| Transcript entry kinds | `.` | + +- **Name clashes are detected at load,** not by reading source: two facets serving one contract name make Chord's + `serve` throw `already served` while the host installs them, and two Durable extensions or task kinds with one name + throw at registry construction. The loader reports the refusing plugin by name. +- **Plugin name clashes** within a scope refuse the later source (order: built-in, global, package, project); project + plugins may not reuse base names. Package clashes surface at install; directory clashes at listing; a global install + that shadows an existing project plugin notifies in both places. To replace a built-in, disable it and use another + name. +- **Each plugin loads on its own:** the loader installs plugins one by one (per-plugin wrapper, as + `PluginRuntime.guard` today); an import error, `setup` throw or name violation becomes a notice naming the plugin. + Plugins that hold its contracts fail at activation with the missing-provider diagnostic (4.2). A broken plugin + never kills a worker or blanks a page. + +### 4.4 Resources + +Like a pi package, a plugin may carry `skills//SKILL.md`, `prompts/*.md` and `themes/*.json` (or list them under +`"p2": { "skills": […], "prompts": […], "themes": […] }` in `package.json`). They follow the facet scope rules: +base resources in every scope, project plugin resources only in their approved scope. Clashes follow existing +precedence (global before project) and are reported. A resource-only plugin needs no code. + +- Project plugin resources are read from the snapshot; a skill's referenced files are in the project directory in the + session's environment, as today. +- Skills of base plugins on remote sessions refer to host files: see 13.2, open question 1. +- Themes reach the TUI picker as data through the scope's TUI bundle. + +## 5. Scopes + +### 5.1 Per surface + +| Surface | Loads | How a change applies | +| --- | --- | --- | +| Server | base scope (server facets) | in place, first host of the rollout (7.5) | +| Session worker | its session's scope (worker facets) | in place; base changes in the rollout, project changes for that worker only (7.4) | +| Web page | the scope of the session it shows; base without a session; base for guests | changed facets in place (D, 14.2); full page load for base code, a scope switch or a stuck generation | +| TUI worker | the same, chosen by the launcher; base for guests | changed facets in place (D); worker swap for base code, a scope switch or a stuck generation | + +A page or TUI worker holds **exactly one scope**: moving between sessions of different scopes is a full page load or +a worker swap. (During a TUI swap the candidate worker is prepared while the old one still runs; they are never both +active.) Between sessions of one scope, navigation stays in-app. A base rollout changes only base facets in a worker; +project facets are installed only by their own per-worker install (7.4). If a base reload removes something a project +plugin depends on, that plugin's consumers fail like any consumer and the scope is marked dirty. + +### 5.2 Scope identity and selection + +- The session catalog stores each session's **scope key** and **revision** (approved snapshot digest, or none). The + server can therefore pick a bundle for a session whose worker is not running. +- **Web:** the server renders `index.html` with a fixed, nonce'd bootstrap script. It takes the session from the URL + (`/s/`) or the tab's sessionStorage route stack, asks `/__p2_web/scope?session=` (authorized per principal: + owners any session, guests their own and always get the base scope), and loads the returned JS and CSS. Selection is + per tab, never a cookie. +- **Session lists carry the scope revision**, so in-app navigation to a session of another scope becomes + `location.assign` (web) or a swap request to the launcher (TUI) instead of an in-app route. +- **TUI:** the manifest takes `?session=` (`/__p2_tui?session=…`, same authorization). The launcher owns the + scope: it fetches the manifest for its current target session and swaps when the hash differs. It caches bundles by + hash (bounded), so switching back is not another download of up to 8 MiB. +- **TUI snapshots** carry drafts, pending send IDs and anchors for every visited identity (`src/tui/snapshot.ts`). + The launcher keeps them **partitioned per scope key**: a swap A→B stores A's snapshot, hands B only the snapshot + last saved for B (if any) with the destination session set explicitly, and never replays A's pending sends into B. + A same-scope revision swap passes the snapshot through, as today. +- **Offline environment:** the worker loads the catalog revision from the local snapshot if present; if not, it loads + the base scope and shows "Project plugins of

unavailable: environment offline". It never substitutes a local + directory and never treats unreadable files as "no plugins". +- **Subagents** with their own cwd use the session's scope; their directory neither confers trust nor loads plugins. + +### 5.3 Push: `ClientBundle` + +Today nothing tells a page or TUI that its bundle is stale: `/version` is a boot ID plus file mtimes, checked only on +`visibilitychange` (`src/hosts/web/main.tsx`); the TUI polls one global manifest. New: a Chord object +`ClientBundle({ session? })` (readable by owners, and by guests for their session) publishes +`{ scopeKey, revision, web, tui }` hashes. A page or TUI holds it for the session it shows; on a hash mismatch the web +reloads (route stack in sessionStorage, drafts in the server's `DraftStore`) and the TUI swaps. Applies to rebuilds, +trust changes, relocations and Reloads started on another device. + +### 5.4 Where project facets may contribute + +Chrome (sidebar, top bar, global sheets, owner notices, footer slots outside the session) is shared by all scopes. +Project web and TUI facets contribute only to session-bound slots: entry and tool renderers, markdown transformers, +the session's composer, sheets and panels, session-bound TUI commands. Registries record the contributing plugin +(new, through the per-plugin wrapper) and refuse project contributions to chrome slots. This is **UX consistency, +not isolation**: project web code can do anything same-origin code can (11). + +## 6. Project trust and snapshots + +### 6.1 Trust decisions + +- `/trust.json` (p2's own, separate from pi's `~/.pi/agent/trust.json`) stores decisions per canonical + directory `(env, path)`; the closest saved decision of the directory or a parent applies, as in pi. +- `defaultProjectTrust`: `"ask"` (default) or `"never"`. There is no `"always"` (A.2). +- Modes per trusted directory: + - **Directory (default):** the owner's Reload (6.3) is the approval of each new revision. + - **Pin to content:** each new revision needs the full trust review again (file list and diffs against the + approved revision); a generic "Reload all" never includes it. +- Approved revisions are stored per scope key; every session with that key uses the latest approved one. +- **Guests never see trust prompts and cannot grant or apply.** Changes are never activated automatically: no + watcher, worker restart, new session or agent tool loads an unapproved revision. + +### 6.2 Snapshots + +A snapshot is an immutable, content-addressed copy of a project's plugin inputs on the p2 host: +`/project-snapshots//` with `plugins/**` and `settings.json` from `/.pi/p2/`. + +- **Capture:** the session worker reads the files through its environment (local or remote, as it reads + `AGENTS.md`), bounded (initial limits: 500 files, 8 MiB total, 1 MiB per file; regular files only, symlinks + refused), and hands them to the server as data. The server lists them (4.1), writes the directory to a + temporary name, makes it read-only and renames it to its digest (sha256 over sorted paths and contents). +- **Approved bytes = executed bytes:** the approval names a digest; workers load worker facets from that directory + and the server builds web and TUI bundles from it. Nothing reads the project directory again for execution. +- **Retention:** snapshots referenced by a catalog session or a trust record are kept; others are collected after + a grace period. + +### 6.3 Change flow + +1. The worker watches `/.pi/p2/` through its environment (`runtime/observe.ts`, including directory creation + and rescans after overflow or reconnect). A change only **marks the scope dirty**; the worker captures a candidate + snapshot. +2. The owner sees "Project plugins changed: Reload" with a **change summary**: files added/removed/changed, facets + added/removed per plugin, plugins disabled by project settings. First-time trust shows the same summary in the + trust notice (directory, plugins, facets, "runs on this device", see 11). +3. **Reload** (owner, interactive web or TUI) approves exactly the candidate's digest: the server builds its web and + TUI bundles (a failed build refuses the apply with a notice; the old revision keeps running), then records the + revision in `trust.json` and the catalog. +4. Each affected session's worker installs the new revision of its project facets **in place** (7.4): running calls + and jobs finish with their old code, new work runs on the new one. Leaving plugins are handled first (10). +5. `ClientBundle` publishes the new hashes; pages and TUIs reload the changed facets in place (a full reload or swap + for a scope switch). + +Revocation and relocation use steps 4–5 with the base scope or the new scope key. A live session never drops to the +base scope mid-turn. Edits arriving during an apply leave the scope dirty again. + +## 7. Loading + +### 7.1 Facets of every surface: loader model D (revised) + +Revision 2 loaded server and worker facets with native `import()` plus Node resolve/load hooks and rejected backend +bundles because a bundle would carry its own copy of Chord. **Superseded by loader model D (14.2):** every facet is one +esbuild bundle whose host packages are **external and passed in as parameters**, so there is exactly one Chord (9.1) +without a module map. Server, worker and TUI worker evaluate it with `vm.Script.runInThisContext`, the web as a +classic ``; + +/** Add a diagnostics bridge without changing the immutable stored source. */ +export function instrumentArtifact(info: ArtifactInfo, content: string): string { + if (info.type === "html") { + const head = /]*)?>/i.exec(content); + if (head) + return `${content.slice(0, head.index + head[0].length)}${DIAGNOSTIC_BOOTSTRAP}${content.slice(head.index + head[0].length)}`; + return `${DIAGNOSTIC_BOOTSTRAP}${content}`; + } + if (info.type === "svg") { + const root = /]*)?>/i.exec(content); + if (root) + return `${content.slice(0, root.index + root[0].length)}${DIAGNOSTIC_BOOTSTRAP}${content.slice(root.index + root[0].length)}`; + } + return content; +} + +export class ArtifactError extends Error { + readonly status: number; + constructor(status: number, message: string) { + super(message); + this.name = "ArtifactError"; + this.status = status; + } +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function validName(name: unknown): name is string { + return typeof name === "string" && name.trim() === name && NAME.test(name); +} + +function validVersion(version: unknown): version is number { + return typeof version === "number" && Number.isSafeInteger(version) && version > 0; +} + +function validTitle(title: unknown): title is string { + return ( + typeof title === "string" && + title.length > 0 && + title.length <= 200 && + title.trim() === title && + title.isWellFormed() && + !/[\p{Cc}\p{Zl}\p{Zp}]/u.test(title) + ); +} + +function validType(type: unknown): type is ArtifactType { + return TYPES.some((candidate) => candidate === type); +} + +function validateInput(value: unknown): ArtifactInput { + if (!record(value) || Object.keys(value).sort().join(",") !== "content,name,title,type") + throw new ArtifactError(400, "Invalid artifact fields"); + if (!validName(value.name)) throw new ArtifactError(400, "Invalid artifact name"); + if (!validTitle(value.title)) throw new ArtifactError(400, "Invalid artifact title (1–200 characters)"); + if (!validType(value.type)) throw new ArtifactError(400, "Invalid artifact type"); + if (typeof value.content !== "string" || !value.content.isWellFormed()) + throw new ArtifactError(400, "Invalid artifact content"); + if (Buffer.byteLength(value.content, "utf8") > MAX_ARTIFACT_BYTES) + throw new ArtifactError(413, "Artifact content exceeds 2 MiB"); + return { name: value.name, title: value.title, type: value.type, content: value.content }; +} + +async function privateDirectory(path: string, create: boolean): Promise { + if (create) { + try { + await mkdir(path, { mode: 0o700 }); + } catch (error) { + if (!record(error) || error.code !== "EEXIST") throw error; + } + } + const stat = await lstat(path); + if (!stat.isDirectory() || (stat.mode & 0o077) !== 0) throw new Error("Invalid private directory"); +} + +/** Private immutable versions. Locators do not authorize access; callers must authorize the session. + * The configured data directory and local filesystem writers are trusted, not a tool sandbox. + */ +export class Artifacts { + readonly directory: string; + private readonly dataDirectory: string; + constructor(dataDirectory: string) { + this.dataDirectory = dataDirectory; + this.directory = join(dataDirectory, "artifacts"); + } + + async save(sessionId: string, input: ArtifactInput): Promise { + if (typeof sessionId !== "string" || sessionId.length !== 36 || !SESSION.test(sessionId)) + throw new ArtifactError(400, "Invalid session"); + const value = validateInput(input); + const session = join(this.directory, sessionId); + const artifact = join(session, value.name); + let temporary: string | undefined; + try { + await mkdir(this.dataDirectory, { recursive: true, mode: 0o700 }); + await privateDirectory(this.dataDirectory, false); + await privateDirectory(this.directory, true); + await privateDirectory(session, true); + await privateDirectory(artifact, true); + let version = 1; + let folder: string; + // Exclusive directory creation claims a number across stores/processes. Never reuse + // abandoned claims: an interrupted version stays absent from readable history. + for (;;) { + folder = join(artifact, String(version)); + try { + await mkdir(folder, { mode: 0o700 }); + break; + } catch (error) { + if (!record(error) || error.code !== "EEXIST") throw error; + if (version === Number.MAX_SAFE_INTEGER) throw new Error("Version limit reached"); + version++; + } + } + const info: ArtifactInfo = { name: value.name, title: value.title, type: value.type, version }; + const serialized = JSON.stringify({ info, content: value.content }); + if (Buffer.byteLength(serialized, "utf8") > MAX_SERIALIZED_BYTES) throw new Error("Invalid serialized size"); + temporary = join(folder, `.version-${randomUUID()}.tmp`); + const output = await open(temporary, "wx", 0o600); + try { + await output.writeFile(serialized, "utf8"); + await output.sync(); + } finally { + await output.close(); + } + // One atomic publication point: metadata and content cannot be observed separately. + await rename(temporary, join(folder, "version.json")); + temporary = undefined; + return info; + } catch { + throw new ArtifactError(500, "Could not save artifact"); + } finally { + if (temporary) await rm(temporary, { force: true }).catch(() => {}); + } + } + + async read(sessionId: string, name: string, version: number): Promise<{ info: ArtifactInfo; content: string }> { + if ( + typeof sessionId !== "string" || + sessionId.length !== 36 || + !SESSION.test(sessionId) || + !validName(name) || + !validVersion(version) + ) + throw new ArtifactError(404, "Artifact not found"); + try { + const session = join(this.directory, sessionId); + const artifact = join(session, name); + const folder = join(artifact, String(version)); + for (const path of [this.dataDirectory, this.directory, session, artifact, folder]) + await privateDirectory(path, false); + const file = await open( + join(folder, "version.json"), + constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW, + ); + let parsed: unknown; + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > MAX_SERIALIZED_BYTES || (stat.mode & 0o077) !== 0) + throw new Error("Invalid artifact file"); + const buffer = Buffer.alloc(Math.min(stat.size + 1, MAX_SERIALIZED_BYTES + 1)); + let size = 0; + for (;;) { + const { bytesRead } = await file.read(buffer, size, buffer.length - size, null); + if (!bytesRead) break; + size += bytesRead; + if (size === buffer.length) throw new Error("Artifact file changed or exceeds limit"); + } + const text = buffer.subarray(0, size).toString("utf8"); + if (!buffer.subarray(0, size).equals(Buffer.from(text, "utf8"))) throw new Error("Invalid UTF-8"); + parsed = JSON.parse(text); + } finally { + await file.close(); + } + if ( + !record(parsed) || + Object.keys(parsed).sort().join(",") !== "content,info" || + !record(parsed.info) || + Object.keys(parsed.info).sort().join(",") !== "name,title,type,version" || + parsed.info.name !== name || + parsed.info.version !== version + ) + throw new Error("Invalid artifact metadata"); + const value = validateInput({ + name: parsed.info.name, + title: parsed.info.title, + type: parsed.info.type, + content: parsed.content, + }); + return { info: { name, title: value.title, type: value.type, version }, content: value.content }; + } catch { + throw new ArtifactError(404, "Artifact not found"); + } + } +} diff --git a/apps/p2/plugins/artifacts/tool.ts b/apps/p2/plugins/artifacts/tool.ts new file mode 100644 index 000000000..80e4dd870 --- /dev/null +++ b/apps/p2/plugins/artifacts/tool.ts @@ -0,0 +1,61 @@ +import { defineDoc, defineExtension, defineTool } from "@earendil-works/pi-durable"; +import { obj, str, union } from "@earendil-works/pi-schema"; +import { literal } from "@p2/app/v1"; +import { type ArtifactInfo, Artifacts } from "./store.ts"; + +type ArtifactState = { artifacts: ArtifactInfo[] }; + +/** Complete artifact index for one conversation; reset/compaction never rewind it. */ +export const ArtifactDoc = defineDoc({ + kind: "p2.artifacts", + version: 1, + scope: "conversation", + history: "latest", + // A latest-only document cannot copy state as of a historical fork point. + // Start forks and independently created subagents with their own empty index. + fork: "initial", + initial: () => ({ artifacts: [] }), + checkpointWhen: (_v, _o, { deltasSinceBase }) => deltasSinceBase >= 64, +}); + +export function createArtifactExtension(dataDirectory: string, sessionId: string) { + const artifacts = new Artifacts(dataDirectory); + return defineExtension({ + name: "p2-artifacts", + tools: [ + defineTool({ + name: "artifact", + description: + "Create or update a session artifact (HTML, SVG, Markdown or text; maximum 2 MiB UTF-8). Reuse its stable lowercase slug name to publish a new immutable numbered version. Versions persist across restarts. All authorized viewers of this session may explicitly click to open an isolated preview; previews never open automatically. Do not publish secrets. HTML may use HTTPS CDN resources, but has no app identity or parent privileges.", + parameters: obj({ + name: str({ pattern: "^[a-z0-9][a-z0-9_-]{0,63}$", minLength: 1, maxLength: 64 }), + title: str({ minLength: 1, maxLength: 200 }), + type: union([literal("html"), literal("svg"), literal("markdown"), literal("text")]), + content: str(), + }), + replay: "unsafe", + execute: async (args, api, context) => { + const info = await artifacts.save(sessionId, args); + await api.commit(async (tx) => { + const state = await tx.doc(ArtifactDoc, api.conversationId); + if ( + !state.artifacts.some( + (artifact) => artifact.name === info.name && artifact.version === info.version, + ) + ) + state.artifacts.push(info); + }, context); + return { + output: [ + { + type: "text" as const, + text: `Artifact ready: ${info.title} (${info.name}, version ${info.version}).`, + }, + ], + details: info, + }; + }, + }), + ], + }); +} diff --git a/apps/p2/plugins/artifacts/tui.ts b/apps/p2/plugins/artifacts/tui.ts new file mode 100644 index 000000000..1ba04b757 --- /dev/null +++ b/apps/p2/plugins/artifacts/tui.ts @@ -0,0 +1,210 @@ +// Artifacts in the TUI (p2-tui-port.md P11c), after the web's artifact card, picker and viewer. The card shows the +// title, type and version; expanded, a text or Markdown artifact shows the content the call wrote. `/artifacts` lists +// the conversation's artifacts (`ArtifactList`): Enter shows one inline (`ArtifactContent`; Markdown rendered, HTML and +// SVG as source), Ctrl+B opens it in the browser (launcher `openUrl`; the browser needs its own sign-in) and Ctrl+S +// downloads it (launcher `download`). The worker never fetches the `/artifact` route itself: it has no cookie. + +import { Markdown, Text } from "@earendil-works/pi-tui"; +import { BACKGROUND_CONTEXT, contribute, defineFacet, follow, statusOf, whenLive } from "@p2/app/v1"; +import type { TuiContext } from "@p2/app/v1/tui"; +import { + getTextOutput, + grantNotice, + ListOverlay, + type ListRow, + lineText, + record, + safeText, + str, + TextViewer, + type ToolRenderers, + type TuiCommand, + TuiCommands, + TuiToolRenderers, +} from "@p2/app/v1/tui"; +import { ArtifactContent, type ArtifactInfo, ArtifactList } from "./contract.ts"; + +const NAME = /^[a-z0-9][a-z0-9_-]{0,63}$/; +/** Collapsed cards show this many lines of a text or Markdown artifact; expanded ones all. */ +const PREVIEW_LINES = 8; +const BROWSER_NOTE = "the browser needs its own sign-in to this server"; + +const message = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +const artifactRenderers: ToolRenderers = { + renderCall(args, theme, context) { + const text = context.lastComponent instanceof Text ? context.lastComponent : new Text("", 0, 0); + const title = str(args.title) || str(args.name) || "..."; + text.setText(`${theme.fg("toolTitle", theme.bold("artifact"))} ${theme.fg("accent", lineText(title))}`); + return text; + }, + renderResult(result, options, theme, context) { + const text = context.lastComponent instanceof Text ? context.lastComponent : new Text("", 0, 0); + const details = record(result.details); + const saved = + !context.isError && + typeof details?.title === "string" && + typeof details.type === "string" && + typeof details.version === "number"; + if (!saved) { + text.setText(`\n${theme.fg(context.isError ? "error" : "toolOutput", getTextOutput(result))}`); + return text; + } + const lines = [ + `${theme.fg("success", "■")} ${lineText(String(details.title))} ${theme.fg("muted", `· ${lineText(String(details.type)).toUpperCase()} · version ${details.version}`)} ${theme.fg("dim", "· /artifacts opens")}`, + ]; + // What the call wrote (text and Markdown; HTML and SVG run only in a browser). + const content = str(context.args.content); + if (content && (details.type === "text" || details.type === "markdown")) { + const all = content.split("\n").map((line) => theme.fg("toolOutput", lineText(line))); + const shown = options.expanded ? all : all.slice(0, PREVIEW_LINES); + lines.push("", ...shown); + if (shown.length < all.length) + lines.push( + theme.fg( + "muted", + `... (${all.length - shown.length} more lines, ${context.ui.keyText("app.tools.expand")} to expand)`, + ), + ); + } + text.setText(`\n${lines.join("\n")}`); + return text; + }, +}; + +/** An artifact shown inline: Markdown rendered, text and HTML/SVG source as lines. */ +function showArtifact(ctx: TuiContext, session: string, artifact: ArtifactInfo): void { + let content: string | undefined; + let status = "Loading…"; + let cache: { width: number; lines: string[] } | undefined; + const lines = (width: number): string[] => { + if (content === undefined) return [ctx.theme.current.fg("muted", status)]; + if (cache?.width === width) return cache.lines; + const body = + artifact.type === "markdown" + ? new Markdown(safeText(content), 0, 0, ctx.theme.markdown).render(width) + : content.split("\n").map((line) => lineText(line)); + const note = + artifact.type === "html" || artifact.type === "svg" + ? [ + ctx.theme.current.fg( + "warning", + `Source only: Ctrl+B in /artifacts opens it in the browser (${BROWSER_NOTE}).`, + ), + "", + ] + : []; + cache = { width, lines: [...note, ...body, ...(status ? ["", ctx.theme.current.fg("warning", status)] : [])] }; + return cache.lines; + }; + const viewer = ctx.overlay( + "artifact-view", + new TextViewer(ctx, { + title: `${artifact.title} · ${artifact.type.toUpperCase()} · version ${artifact.version}`, + lines, + onClose: () => { + ref.release(); + viewer.close(); + }, + }), + { width: "100%", anchor: "bottom-center" }, + ); + const ref = ctx.host.hold(ArtifactContent, { session }, BACKGROUND_CONTEXT); + void whenLive(ref) + .then(async () => { + if (statusOf(ref) !== "live" || ref.variant !== "reader") throw new Error("Artifacts are not available"); + const read = await ref.read({ name: artifact.name, version: artifact.version }, BACKGROUND_CONTEXT); + status = read.truncated ? "… truncated: Ctrl+S in /artifacts downloads all of it" : ""; + content = read.content; + }) + .catch((error: unknown) => { + status = message(error); + }) + .finally(() => { + cache = undefined; + ctx.requestRender(); + }); +} + +function openArtifacts(ctx: TuiContext): void { + const session = ctx.identity.session; + if (!session) return; + const list = ctx.host.hold(ArtifactList, { session, conversation: ctx.identity.conversation }, BACKGROUND_CONTEXT); + let artifacts: ArtifactInfo[] = []; + const stop = follow(list.artifacts, (value) => { + artifacts = (value ?? []).filter((artifact) => NAME.test(artifact.name)); + ctx.requestRender(); + }); + const byRow = (row: ListRow | undefined) => artifacts.find((artifact) => artifact.name === row?.key); + const overlay = ctx.overlay( + "artifacts", + new ListOverlay(ctx, { + title: () => `Artifacts (${artifacts.length})`, + rows: () => + artifacts.map((artifact) => ({ + key: artifact.name, + label: artifact.title, + detail: `${artifact.type.toUpperCase()} · v${artifact.version}`, + })), + onSelect: (row) => { + const artifact = byRow(row); + if (artifact) showArtifact(ctx, session, artifact); + }, + actions: [ + { + key: "ctrl+b", + label: "open in browser", + run: (row) => { + const artifact = byRow(row); + if (!artifact) return; + const query = new URLSearchParams({ + session, + name: artifact.name, + version: String(artifact.version), + }); + void ctx + .grant("openUrl", { kind: "origin", path: `/artifact?${query}` }) + .then((result) => grantNotice(ctx, `Opened ${artifact.title} in the browser`, result)); + }, + }, + { + key: "ctrl+s", + label: "download", + run: (row) => { + const artifact = byRow(row); + if (!artifact) return; + void ctx + .grant("download", { kind: "artifact", session, name: artifact.name, version: artifact.version }) + .then((result) => grantNotice(ctx, `Downloaded ${artifact.title}`, result)); + }, + }, + ], + footer: () => [ + ctx.theme.current.fg("dim", ` Enter shows it here; Ctrl+B opens it in the browser (${BROWSER_NOTE})`), + ], + onClose: () => { + stop(); + list.release(); + overlay.close(); + }, + }), + { width: "100%", anchor: "bottom-center" }, + ); +} + +const COMMANDS: Record = { + "p2.artifacts": { + title: "Artifacts", + category: "Session", + when: "chat && !overlay", + slash: { name: "artifacts" }, + run: (ctx) => openArtifacts(ctx), + }, +}; + +export default defineFacet((host) => + contribute(host, "p2.tui-artifacts", (self) => { + self.contribute(TuiToolRenderers, { artifact: artifactRenderers }); + self.contribute(TuiCommands, COMMANDS); + }), +); diff --git a/apps/p2/plugins/artifacts/web.tsx b/apps/p2/plugins/artifacts/web.tsx new file mode 100644 index 000000000..0556252ac --- /dev/null +++ b/apps/p2/plugins/artifacts/web.tsx @@ -0,0 +1,111 @@ +import { compile, obj, union } from "@earendil-works/pi-schema"; +import { contribute, defineFacet, literal } from "@p2/app/v1"; +import { + ArtifactIcon, + ArtifactPicker, + closeLayers, + FilePanel, + OpenViewer, + openLayer, + SessionId, + Slots, + useObject, + useReadable, + useRoute, + type ViewerItem, +} from "@p2/app/v1/web"; +import type { ReactNode } from "react"; +import { DownloadInfoSchema } from "../files/contract.ts"; +import { ArtifactInfoSchema, ArtifactList } from "./contract.ts"; +import { ArtifactPanel } from "./preview.tsx"; + +const ViewerSchema = union([ + obj({ kind: literal("file"), file: DownloadInfoSchema }, { open: true }), + obj({ kind: literal("artifact"), artifact: ArtifactInfoSchema }, { open: true }), +]); +const isViewer = compile(ViewerSchema).check; + +/** Untrusted URL metadata is only a locator. HTTP routes authorize every content fetch. */ +function viewer(overlay: string | undefined): ViewerItem | undefined { + if (!overlay?.startsWith("viewer:")) return undefined; + try { + const value: unknown = JSON.parse(overlay.slice("viewer:".length)); + return isViewer(value) ? value : undefined; + } catch { + return undefined; + } +} + +function Content({ children }: { children?: ReactNode }) { + const route = useRoute(); + const list = useObject( + ArtifactList, + route.session ? { session: route.session, conversation: route.conversation ?? 1 } : undefined, + ); + const artifacts = useReadable(list?.artifacts) ?? []; + + const top = route.layers.at(-1); + const item = viewer(top?.kind === "viewer" && top.name === "artifacts" ? top.locator : undefined); + const open = (item: ViewerItem): void => + openLayer({ kind: "viewer", name: "artifacts", locator: `viewer:${JSON.stringify(item)}` }); + return ( + + + {children} + {top?.kind === "sheet" && top.name === "artifacts" && ( + open({ kind: "artifact", artifact })} + onClose={closeLayers} + /> + )} + + {route.session && + item && + (item.kind === "artifact" ? ( + + ) : ( + + ))} + + + ); +} + +function ArtifactButton() { + const route = useRoute(); + const list = useObject( + ArtifactList, + route.session ? { session: route.session, conversation: route.conversation ?? 1 } : undefined, + ); + const artifacts = useReadable(list?.artifacts) ?? []; + const count = artifacts.length; + if (!count) return null; + return ( + + ); +} + +export default defineFacet((host) => { + contribute(host, "p2.web-artifacts", (self) => { + self.contribute(Slots, { "chat-content": Content, "artifact-actions": ArtifactButton }); + }); +}); diff --git a/apps/p2/plugins/artifacts/worker.ts b/apps/p2/plugins/artifacts/worker.ts new file mode 100644 index 000000000..b52d0a16f --- /dev/null +++ b/apps/p2/plugins/artifacts/worker.ts @@ -0,0 +1,100 @@ +import type { ConversationId } from "@earendil-works/pi-durable"; +import { arr, obj } from "@earendil-works/pi-schema"; +import { + type Context, + canView, + contribute, + defineFacet, + defineTypedState, + deriveTyped, + forbidden, + signalOf, +} from "@p2/app/v1"; +import { CommitSignal, Extensions, harnessOf, workerOf } from "@p2/app/v1/worker"; +import { ArtifactContent, ArtifactInfoSchema, ArtifactList, MAX_ARTIFACT_PREVIEW } from "./contract.ts"; +import { Artifacts } from "./store.ts"; +import { ArtifactDoc, createArtifactExtension } from "./tool.ts"; + +/** The artifacts of one conversation (in memory, from its document). */ +const Listed = defineTypedState("p2.artifact-list", { + schema: obj({ artifacts: arr(ArtifactInfoSchema) }), + version: 1, + initial: () => ({ artifacts: [] }), +}); + +export default defineFacet((host) => { + contribute(host, "p2.artifact-tools", (self) => { + const worker = workerOf(self); + // The tool's declaration; each call saves for the worker's session. + const template = createArtifactExtension("", ""); + self.contribute(Extensions, { + [template.name]: { + ...template, + tools: template.tools!.map((tool) => ({ + ...tool, + execute: (...args: Parameters) => + createArtifactExtension(worker().init.dataDir, worker().init.values.session.id).tools![0]!.execute( + ...args, + ), + })), + }, + }); + }); + + host.serve(ArtifactContent, ({ session }, principal, self) => { + if (!canView(principal, session)) throw forbidden("viewers only"); + const worker = workerOf(self); + return { + variant: "reader", + members: { + read: async ({ name, version }) => { + const { info, content } = await new Artifacts(worker().init.dataDir).read(session, name, version); + const truncated = content.length > MAX_ARTIFACT_PREVIEW; + // Never split a surrogate pair at the cut. + let end = MAX_ARTIFACT_PREVIEW; + if (truncated && /[\uD800-\uDBFF]/.test(content[end - 1] ?? "")) end--; + return { info, content: truncated ? content.slice(0, end) : content, truncated }; + }, + }, + }; + }); + + host.serve(ArtifactList, ({ session, conversation }, principal, self) => { + if (!canView(principal, session)) throw forbidden("viewers only"); + const harness = harnessOf(self); + const id = conversation as ConversationId; + const listed = self.state(Listed); + const show = async (ctx: Context) => { + const doc = await harness().durable.snapshot(ArtifactDoc, id, ctx); + await listed.change((draft) => { + // A draft's arrays are mutable; the document's are the same JSON. + draft.artifacts = [...(doc?.artifacts ?? [])] as typeof draft.artifacts; + }, ctx); + }; + const commits = self.open( + async (ctx) => { + const durable = harness().durable; + if (!(await durable.conversation(id, ctx))) throw new Error(`Unknown conversation ${conversation}`); + await show(ctx); + return new CommitSignal(durable, (publication) => + publication.changes.some( + (change) => + change.type === "document" && + change.record.kind === ArtifactDoc.definition.kind && + change.conversationId === conversation, + ), + ); + }, + (signal) => signal.close(), + ); + self.onLive(async (ctx) => { + const signal = signalOf(ctx); + while (!signal.aborted) { + const seen = commits.value.version; + await show(ctx); + await commits.value.next(seen, signal).catch(() => {}); + } + }); + return { variant: "reader", members: { artifacts: deriveTyped(self, [listed], ([value]) => value.artifacts) } }; + }); +}); diff --git a/apps/p2/plugins/chat-mode/contract.ts b/apps/p2/plugins/chat-mode/contract.ts new file mode 100644 index 000000000..4038037db --- /dev/null +++ b/apps/p2/plugins/chat-mode/contract.ts @@ -0,0 +1,136 @@ +// Chat mode: the people viewing a conversation talk to each other in its transcript. The agent reads along only +// when someone mentions it. Served by each session worker. + +import { arr, bool, compile, int, num, obj, opt, type Static, str, union } from "@earendil-works/pi-schema"; +import { literal, local, MessageDataSchema, method, param, propertiesOf, remote, sessionHost } from "@p2/app/v1"; + +/** Entry kind of a chat message: display-only, it carries no model message. */ +export const CHAT_ENTRY = "chat.message"; +/** Entry kind of a mention: an input whose model message is the digest, and whose data is what the person wrote. */ +export const MENTION_ENTRY = "chat.mention"; +/** + * The message a reply answers: an entry of the same conversation, with its author and the start of its text. The + * worker derives author and excerpt from that entry; a client only names the entry. Pi's answers can be replied to. + */ +export const ReplyToSchema = obj( + { + entryId: int({ minimum: 1 }), + author: obj( + { + kind: union([literal("owner"), literal("guest"), literal("agent")]), + id: str(), + name: str(), + }, + { open: true }, + ), + excerpt: str(), + }, + { open: true }, +); +export type ReplyTo = Static; + +/** + * A chat entry's data: a person's message (author stamped by the worker, attachments as the session's upload IDs), + * perhaps answering another. + */ +export const ChatMessageSchema = obj( + { + ...propertiesOf(MessageDataSchema), + replyTo: opt(ReplyToSchema), + }, + { open: true }, +); +export type ChatMessage = Static; + +/** A mention entry's data: what the person wrote, and the IDs of the chat entries its digest passed on to Pi. */ +export const MentionSchema = obj( + { + ...propertiesOf(ChatMessageSchema), + included: arr(int()), + }, + { open: true }, +); +export type Mention = Static; + +const isChatMessage = compile(ChatMessageSchema).check; +const isMention = compile(MentionSchema).check; + +/** Entries are history: read their data defensively, never assume this code wrote it. */ +export const chatMessage = (data: unknown): ChatMessage | undefined => (isChatMessage(data) ? data : undefined); +export const mention = (data: unknown): Mention | undefined => (isMention(data) ? data : undefined); + +/** + * The name people call the agent by in a conversation (chat mode's setting): one word, as it is spoken and mentioned + * ("Pi", "Claude"). Letters, digits, `_` and `-`, starting with a letter. + */ +export const DEFAULT_AGENT_NAME = "Pi"; +export const AgentNameSchema = str({ pattern: "^[A-Za-z][A-Za-z0-9_-]{0,31}$" }); + +/** The mention of the agent called `name`: `@` and the name in lower case ("@pi", "@claude"). */ +export const mentionToken = (name: string): string => `@${name.toLowerCase()}`; + +/** The name as a regular expression, matched literally (a valid name holds no special character; `-` is literal). */ +const escapeName = (name: string): string => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +/** + * `@` in any case, as a word of its own: with the default, "@pi", "@Pi," and "(@PI)" mention; "@pizza" and "me@pi" + * do not. + */ +export const mentionPattern = (name = DEFAULT_AGENT_NAME): RegExp => + new RegExp(`(? mentionPattern(name).test(text); + +/** + * The agent's name spoken anywhere in an utterance, as speech recognition writes it (no "@"): a whole word in any case, + * whatever punctuation surrounds it. With the default: "Hey Pi, …", "can Pi summarize this", "what do you think, + * Pi?". Not "pie", "piano", "pip" or "Pi-hole"; and not "Pi's": a possessive or contraction talks about the agent more + * often than to it ("Pi's answer was odd"), and is cheap to say again. + */ +const spokenName = (name: string): RegExp => + new RegExp(`(? EXCERPT_LENGTH ? `${flat.slice(0, EXCERPT_LENGTH - 1).trimEnd()}…` : flat; +} + +const AGENT = { kind: "agent", id: "agent", name: "Pi" } as const; + +/** + * What a reply quotes from an entry, derived from the entry's own data: a chat message, a person's input (for a + * mention, what they wrote, not the digest), or one of Pi's answers. Undefined for anything else. + */ +export function quoteOf(entry: EntryRecord): ReplyTo | undefined { + const quote = (author: ReplyTo["author"], text: string): ReplyTo | undefined => { + const result = { entryId: entry.id, author, excerpt: excerpt(text) }; + return isReplyTo(result) ? result : undefined; + }; + const message = entry.model?.[0]; + if (message?.role === "assistant") + return quote(AGENT, message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join(" ")); + const person = messageOf(entry.data, entry.kind === "pi.user" && message ? message.content : undefined); + return person && quote(person.author, person.text); +} + +/** A chat entry: its ID (what a mention records) and its message. */ +export type ChatEntry = { readonly id: number; readonly message: ChatMessage }; + +/** + * Where a scan may stop: the lowest chat entry the latest placed mention passed on. Its digest took every chat message + * not yet passed on, so older ones were passed on before, or dropped by the digest's cap. A mention that passed on + * nothing bounds nothing: chat placed while it waited lies below it. + */ +const boundaryOf = (entries: readonly EntryRecord[]): number | undefined => { + for (const entry of entries) { + const data = entry.kind === MENTION_ENTRY ? mention(entry.data) : undefined; + if (data?.included.length) return Math.min(...data.included); + } + return undefined; +}; + +/** Whether a newest-first read has gone far enough for `unmentioned` (the reader may stop). */ +export const readEnough = (entries: readonly EntryRecord[]): boolean => { + const boundary = boundaryOf(entries); + return boundary !== undefined && (entries.at(-1)?.id ?? 0) <= boundary; +}; + +/** + * Chat messages no mention has passed on to Pi yet, oldest first. `entries` are newest first, as a backwards scan + * reads them; `inbox` holds mentions still queued. Each mention records the chat entries its digest included, so a chat + * message placed while a mention waited in the queue is not covered by it and goes with the next one. The scan stops + * at the lowest chat entry the latest mention passed on (see `boundaryOf`). + */ +export function unmentioned(entries: readonly EntryRecord[], inbox?: InboxState): ChatEntry[] { + const covered = new Set(); + for (const item of inbox?.items ?? []) { + const queued = item.mode !== "write" && item.entry?.kind === MENTION_ENTRY ? mention(item.entry.data) : undefined; + for (const id of queued?.included ?? []) covered.add(id); + } + const boundary = boundaryOf(entries) ?? Number.NEGATIVE_INFINITY; + const found: ChatEntry[] = []; + for (const entry of entries) { + if (entry.id < boundary) break; + const earlier = entry.kind === MENTION_ENTRY ? mention(entry.data) : undefined; + for (const id of earlier?.included ?? []) covered.add(id); + const message = entry.kind === CHAT_ENTRY ? chatMessage(entry.data) : undefined; + if (message && !covered.has(entry.id)) found.push({ id: entry.id, message }); + } + return found.reverse(); +} + +/** The mention's text for the model: the chat not passed on yet, with its authors, then who asks and what. */ +export function digest(chat: readonly ChatEntry[], author: Author, text: string, answering?: ReplyTo): string { + const quoted = chat.slice(-MAX_DIGEST_MESSAGES); + const omitted = chat.length - quoted.length; + const lines = quoted.map(({ message }) => { + const files = message.attachments.length + ? ` [attached: ${message.attachments.map((file) => file.name).join(", ")}]` + : ""; + return `${speaker(message.author, message.replyTo)}: ${message.text}${files}`; + }); + const header = chat.length + ? [ + "", + "Since you were last mentioned, people in this session talked to each other. They did not address you:", + ...(omitted ? [`(${omitted} earlier messages omitted)`] : []), + ...lines, + "", + "", + ] + : []; + return [...header, `${speaker(author, answering)} mentioned you:`, text].join("\n"); +} + +/** Upload IDs to resolve for a mention: the chat's newest attachments, then the asker's own. */ +export const digestFiles = (chat: readonly ChatEntry[], own: readonly string[]): string[] => [ + ...chat + .flatMap(({ message }) => message.attachments.map((file) => file.id)) + .slice(-MAX_DIGEST_FILES) + .filter((id) => !own.includes(id)), + ...own, +]; + +/** + * A mention as the chat plugin submits it: the one place that decides what a mention is. It is placed as a + * `chat.mention` entry whose data is what the person wrote (author, text, attachments, reply) and the chat entries + * passed on; the model reads the digest, which the chat plugin starts with the author's speaker block. + */ +export function composeMention(input: { + readonly author: Author; + readonly text: string; + readonly chat: readonly ChatEntry[]; + readonly replyTo?: ReplyTo; + readonly attachments: readonly { readonly id: string; readonly name: string }[]; + readonly files: readonly Attachment[]; + readonly requestId: string; + readonly whenBusy?: "steer" | "followUp"; +}): Submit { + const data: Mention = { + author: input.author, + text: input.text, + attachments: input.attachments.map(({ id, name }) => ({ id, name })), + ...(input.replyTo ? { replyTo: input.replyTo } : {}), + included: input.chat.slice(-MAX_DIGEST_MESSAGES).map((entry) => entry.id), + }; + return { + text: digest(input.chat, input.author, input.text, input.replyTo), + requestId: input.requestId, + ...(input.whenBusy ? { whenBusy: input.whenBusy } : {}), + author: input.author, + files: [...input.files], + maxImages: MAX_DIGEST_IMAGES, + entry: { kind: MENTION_ENTRY, data }, + }; +} diff --git a/apps/p2/plugins/chat-mode/display.ts b/apps/p2/plugins/chat-mode/display.ts new file mode 100644 index 000000000..0e23c86bc --- /dev/null +++ b/apps/p2/plugins/chat-mode/display.ts @@ -0,0 +1,18 @@ +// How chat entries add to the transcript's display of people's messages (the shared `TranscriptEntries` registry): +// a reply's quoted line and a mention's note. Pure; used by the web and the TUI facets. +import type { EntryDisplay } from "../chat/contract.ts"; +import { CHAT_ENTRY, chatMessage, MENTION_ENTRY, mention, type ReplyTo } from "./contract.ts"; + +/** A reply's quoted line, from the quote the worker stored with it. */ +const quoted = (replyTo: ReplyTo | undefined) => + replyTo && { replyTo: { entryId: replyTo.entryId, name: replyTo.author.name, excerpt: replyTo.excerpt } }; +const plural = (count: number): string => (count === 1 ? "1 chat message" : `${count} chat messages`); + +export const CHAT_ENTRY_DISPLAYS: Record EntryDisplay | undefined> = { + [CHAT_ENTRY]: (data) => quoted(chatMessage(data)?.replyTo), + [MENTION_ENTRY]: (data) => { + const asked = mention(data); + const count = asked?.included.length ?? 0; + return asked && { ...quoted(asked.replyTo), ...(count ? { note: `with ${plural(count)}` } : {}) }; + }, +}; diff --git a/apps/p2/plugins/chat-mode/package.json b/apps/p2/plugins/chat-mode/package.json new file mode 100644 index 000000000..9b7ed27ae --- /dev/null +++ b/apps/p2/plugins/chat-mode/package.json @@ -0,0 +1,5 @@ +{ + "name": "chat-mode", + "private": true, + "type": "module" +} diff --git a/apps/p2/plugins/chat-mode/style.css b/apps/p2/plugins/chat-mode/style.css new file mode 100644 index 000000000..e1bdb19e8 --- /dev/null +++ b/apps/p2/plugins/chat-mode/style.css @@ -0,0 +1,11 @@ +/* Chat messages never reached the agent: an outlined bubble instead of a filled one. */ +.chat-message .bubble { + border: 1px dashed var(--line); + background: transparent; +} +/* How the agent is mentioned while chat mode is on; the owner's opens its renaming. */ +.chat-mode-name { + padding: 0 var(--s1); + font-family: var(--mono); + font-size: var(--fs-xs); +} diff --git a/apps/p2/plugins/chat-mode/tui.ts b/apps/p2/plugins/chat-mode/tui.ts new file mode 100644 index 000000000..e182b227c --- /dev/null +++ b/apps/p2/plugins/chat-mode/tui.ts @@ -0,0 +1,165 @@ +// Chat mode in the TUI (p2-tui-port.md P11b), after the web facet: chat and mention entries show their reply quote +// and note (the shared `TranscriptEntries` display), a composer chip shows the mode (the owner toggles it with +// `/chat-mode`), and while it is on `/reply` picks the message the next send answers and `@pi` completes first. + +import type { Held } from "@p2/app/v1"; +import { BACKGROUND_CONTEXT, contribute, defineFacet, liveIn, statusOf } from "@p2/app/v1"; +import type { TuiContext } from "@p2/app/v1/tui"; +import { + chatSurface, + ListOverlay, + type TuiCommand, + TuiCommands, + type TuiRuntime, + TuiScope, + TuiSlots, + type TuiSurface, +} from "@p2/app/v1/tui"; +import { TranscriptEntries } from "../chat/contract.ts"; +import { ChatMode, DEFAULT_AGENT_NAME, mentionToken } from "./contract.ts"; +import { CHAT_ENTRY_DISPLAYS } from "./display.ts"; + +const hint = (name: string) => `messages go to the people here; ${mentionToken(name)} asks ${name}`; + +/** The `composer:chat-mode` chip: the viewed conversation's chat mode, and the reply switch of the chat surface. */ +export class ChatModeChip implements TuiSurface { + readonly runtime: TuiRuntime; + private scope: TuiScope | undefined; + private identity: string | undefined; + private mode: () => Held | undefined = () => undefined; + enabled = false; + /** The agent's name in this conversation (what people mention). */ + name = DEFAULT_AGENT_NAME; + private waiting = 0; + private stop: () => void; + constructor(runtime: TuiRuntime) { + this.runtime = runtime; + const changed = (): void => { + this.syncReplies(); + if (runtime.identity === this.identity) return; + this.identity = runtime.identity; + this.scope?.dispose(); + this.scope = undefined; + this.mode = () => undefined; + this.enabled = false; + this.waiting = 0; + this.name = DEFAULT_AGENT_NAME; + const session = runtime.session; + if (!session) return; + this.scope = new TuiScope(runtime); + this.mode = this.scope.watch(ChatMode, { session, conversation: runtime.conversation }, (ref, follow) => { + follow(ref.enabled, (enabled) => { + this.enabled = enabled ?? false; + }); + follow(ref.waiting, (waiting) => { + this.waiting = waiting ?? 0; + }); + follow(ref.name, (name) => { + this.name = name ?? DEFAULT_AGENT_NAME; + }); + }); + }; + runtime.listeners.add(changed); + this.stop = () => runtime.listeners.delete(changed); + changed(); + } + /** The owner's live reference (to switch the mode). */ + owner(): Held | undefined { + const ref = this.mode(); + return liveIn(ref, "owner"); + } + private live(): boolean { + const ref = this.mode(); + return ref !== undefined && statusOf(ref) === "live"; + } + /** Replies belong to chat mode (the web's `setCanReply`). */ + private syncReplies(): void { + chatSurface(this.runtime.ctx)?.setCanReply(this.live() && this.enabled, this.name); + } + render(): string[] { + const theme = this.runtime.theme.current; + if (!this.live()) return []; + const label = this.enabled && this.waiting ? `Chat · ${this.waiting}` : "Chat"; + if (this.enabled) + return [`${theme.bg("selectedBg", theme.fg("accent", ` ${label} `))} ${theme.fg("dim", hint(this.name))}`]; + // Off: only the owner sees the switch. + return this.owner() ? [theme.fg("dim", `[${label} off · /chat-mode]`)] : []; + } + invalidate(): void {} + dispose(): void { + this.stop(); + this.scope?.dispose(); + } +} + +const CHIP = "composer:chat-mode"; +const chipOf = (ctx: TuiContext): ChatModeChip | undefined => { + const surface = ctx.surface(CHIP); + return surface instanceof ChatModeChip ? surface : undefined; +}; + +/** `/reply`: the message the next send answers (a pick list, newest first); again with a target set, cancels. */ +function reply(ctx: TuiContext): void { + const chat = chatSurface(ctx); + if (!chat) return; + if (chat.replyTo) { + chat.setReplyTo(undefined); + ctx.notify("Reply cancelled"); + return; + } + const targets = chat.replyTargets(); + const handle = ctx.overlay( + "reply", + new ListOverlay(ctx, { + title: "Reply to", + rows: () => + targets.map((target) => ({ + key: String(target.entryId), + label: `${target.name}: ${target.excerpt}`, + })), + onSelect: (row) => { + handle.close(); + const target = targets.find((candidate) => String(candidate.entryId) === row.key); + if (target) chat.setReplyTo({ entryId: target.entryId, name: target.name }); + }, + onClose: () => handle.close(), + }), + { width: "100%", anchor: "bottom-center" }, + ); +} + +export const CHAT_MODE_COMMANDS: Record = { + "p2.chatMode.toggle": { + title: "Toggle chat mode", + category: "Chat", + when: "chat && owner && !overlay", + slash: { name: "chat-mode" }, + run: async (ctx) => { + const chip = chipOf(ctx); + const owner = chip?.owner(); + if (!chip || owner?.variant !== "owner") { + ctx.notify("Chat mode is not available here", "warning"); + return; + } + // The chip follows the state as it arrives; the notice says what was asked for. + const enabled = !chip.enabled; + await owner.set({ enabled }, BACKGROUND_CONTEXT); + ctx.notify(enabled ? `Chat mode on: ${hint(chip.name)}` : "Chat mode off"); + }, + }, + "p2.chatMode.reply": { + title: "Reply to a message (chat mode)", + category: "Chat", + when: "chat && participant && canReply && !overlay", + slash: { name: "reply" }, + run: (ctx) => reply(ctx), + }, +}; + +export default defineFacet((host) => + contribute(host, "p2.tui-chat-mode", (self) => { + self.contribute(TranscriptEntries, CHAT_ENTRY_DISPLAYS); + self.contribute(TuiSlots, { [CHIP]: (runtime: TuiRuntime): TuiSurface => new ChatModeChip(runtime) }); + self.contribute(TuiCommands, CHAT_MODE_COMMANDS); + }), +); diff --git a/apps/p2/plugins/chat-mode/web.tsx b/apps/p2/plugins/chat-mode/web.tsx new file mode 100644 index 000000000..c4b1946ec --- /dev/null +++ b/apps/p2/plugins/chat-mode/web.tsx @@ -0,0 +1,158 @@ +// Chat mode in the browser: the composer's "Chat" chip, and chat entries in the transcript. While chat mode is on, a +// second chip shows how the agent is mentioned (`@pi`); the owner taps it to rename the agent for this conversation +// (what people type after `@`, and what voice mode listens for). + +import { BACKGROUND_CONTEXT, defineFacet } from "@p2/app/v1"; +import { Sheet, Slots, useComposerState, useObject, useReadable, useRoute, useStatus } from "@p2/app/v1/web"; +import { useEffect, useState } from "react"; +import { TranscriptEntries } from "../chat/contract.ts"; +import { ChatMode, DEFAULT_AGENT_NAME, mentionToken } from "./contract.ts"; +import { CHAT_ENTRY_DISPLAYS } from "./display.ts"; +import "./style.css"; +import { contribute } from "@p2/app/v1"; + +const hint = (name: string) => + `Chat mode: messages go to the people in this session, not to ${name}. Mention ${mentionToken(name)} (or say "${name}") to ask ${name}; it then reads the chat since its last answer.`; + +/** One word, as typed after `@` and spoken: letters, digits, `_` and `-`, starting with a letter (AgentNameSchema). */ +const VALID_NAME = /^[A-Za-z][A-Za-z0-9_-]{0,31}$/; + +/** The owner's sheet for the agent's name in this conversation. */ +function NameSheet({ + name, + onSave, + onClose, +}: { + name: string; + onSave: (name: string) => Promise; + onClose: () => void; +}) { + const [draft, setDraft] = useState(name); + const [error, setError] = useState(); + const valid = VALID_NAME.test(draft.trim()); + const save = () => { + const next = draft.trim(); + if (!VALID_NAME.test(next)) return; + void onSave(next) + .then(onClose) + .catch((failure: unknown) => setError(failure instanceof Error ? failure.message : String(failure))); + }; + return ( + +
{ + event.preventDefault(); + save(); + }} + > + +

+ People mention it as {mentionToken(valid ? draft.trim() : name)}, and in voice mode its name anywhere in + what they say counts. One word: letters, digits, - and _. +

+ {error &&

{error}

} +
+ + {name !== DEFAULT_AGENT_NAME && ( + + )} +
+
+ + ); +} + +/** Beside Steer in the composer: the owner's switch; for guests, a marker while chat mode is on. */ +function ChatModeChip() { + const route = useRoute(); + const { session } = route; + const mode = useObject(ChatMode, session ? { session, conversation: route.conversation ?? 1 } : undefined); + const live = useStatus(mode) === "live"; + const enabled = useReadable(mode?.enabled) ?? false; + const waiting = useReadable(mode?.waiting) ?? 0; + const name = useReadable(mode?.name) ?? DEFAULT_AGENT_NAME; + const [renaming, setRenaming] = useState(false); + const label = enabled && waiting ? `Chat · ${waiting}` : "Chat"; + // Replies belong to chat mode: the transcript offers them while it is on, and the composer offers the mention. + const { setCanReply, setReplyTo, setMentionName } = useComposerState(); + const replies = live && enabled; + useEffect(() => { + setCanReply(replies); + if (!replies) setReplyTo(undefined); + return () => setCanReply(false); + }, [replies, setCanReply, setReplyTo]); + useEffect(() => { + setMentionName(name); + return () => setMentionName(DEFAULT_AGENT_NAME); + }, [name, setMentionName]); + if (!mode || !live) return null; + const title = hint(name); + if (mode.variant !== "owner") + return enabled ? ( + <> + + {label} + + + {mentionToken(name)} + + + ) : null; + return ( + <> + + {enabled && ( + + )} + {renaming && ( + mode.rename({ name: next }, BACKGROUND_CONTEXT)} + onClose={() => setRenaming(false)} + /> + )} + + ); +} + +export default defineFacet((host) => { + contribute(host, "p2.chat-mode-web", (self) => { + self.contribute(Slots, { "composer:chat-mode": ChatModeChip }); + self.contribute(TranscriptEntries, CHAT_ENTRY_DISPLAYS); + }); +}); diff --git a/apps/p2/plugins/chat-mode/worker.ts b/apps/p2/plugins/chat-mode/worker.ts new file mode 100644 index 000000000..2bfc49044 --- /dev/null +++ b/apps/p2/plugins/chat-mode/worker.ts @@ -0,0 +1,254 @@ +// Chat mode (worker): the per-conversation switch and the agent's name (what people mention, `@pi` by default), kept +// in a Durable document; the router that turns messages into chat entries or mentions; and the view each browser +// holds. + +import { + type ConversationId, + type Harness as DurableHarness, + defineDoc, + type EntryId, + type EntryRecord, + InboxDoc, +} from "@earendil-works/pi-durable"; +import { bool, num, obj } from "@earendil-works/pi-schema"; +import { + type Context, + type Ctx, + canView, + contribute, + defineFacet, + defineState, + forbidden, + invalid, + isOwner, + signalOf, +} from "@p2/app/v1"; +import { durableContext } from "@p2/app/v1/node"; +import { CommitSignal, harnessOf } from "@p2/app/v1/worker"; +import { Conversation, SendRouters } from "../chat/contract.ts"; +import { Attachments, MAX_ATTACHMENTS } from "../files/contract.ts"; +import { + AgentNameSchema, + CHAT_ENTRY, + type ChatMessage, + ChatMode, + ChatModeSetting, + DEFAULT_AGENT_NAME, + mentionsAgent, + type ReplyTo, +} from "./contract.ts"; +import { composeMention, digestFiles, quoteOf, readEnough, unmentioned } from "./digest.ts"; + +/** + * The switch and the agent's name, stored in the conversation (a fork or a new subagent starts with chat mode off and + * the default name). + */ +export const ChatModeDoc = defineDoc<{ enabled: boolean; name: string }>({ + kind: "chat-mode.settings", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ enabled: false, name: DEFAULT_AGENT_NAME }), + checkpointWhen: () => true, + // Version 1 had the switch only. + migrate: (value) => ({ enabled: value.enabled === true, name: DEFAULT_AGENT_NAME }), +}); + +/** A conversation's newest entries, newest first, until `unmentioned` has what it needs (at most `limit`). */ +async function recentEntries(durable: DurableHarness, id: ConversationId, ctx: Context, limit = 2000) { + const conversation = await durable.conversation(id, durableContext(ctx)); + const entries: EntryRecord[] = []; + let before = Number.MAX_SAFE_INTEGER; + while (conversation && entries.length < limit && !readEnough(entries)) { + const page = await conversation.entries( + { maxEntryId: (before - 1) as EntryId }, + 256, + undefined, + durableContext(ctx), + ); + for (const entry of page.items) { + entries.push(entry); + before = entry.id; + } + if (page.next === undefined) break; + } + return entries; +} + +/** + * What a reply quotes, read from the entry itself. Only an entry of this conversation qualifies: Durable lists a + * conversation's own entries, so an ID from another conversation is not found. + */ +export async function replyTarget(durable: DurableHarness, id: ConversationId, entryId: number, ctx: Context) { + const conversation = await durable.conversation(id, durableContext(ctx)); + const only = { minEntryId: entryId as EntryId, maxEntryId: entryId as EntryId }; + const page = await conversation?.entries(only, 1, undefined, durableContext(ctx)); + const entry = page?.items[0]; + const quote: ReplyTo | undefined = entry?.id === entryId ? quoteOf(entry) : undefined; + if (!quote) throw invalid("args", "That message cannot be replied to"); + return quote; +} + +/** What the cells show of the document, and how many messages wait for a mention. */ +const Shown = defineState("p2.chat-mode-setting", { + schema: obj({ enabled: bool(), name: AgentNameSchema, waiting: num() }), + version: 1, + initial: () => ({ enabled: false, name: DEFAULT_AGENT_NAME, waiting: 0 }), +}); + +export default defineFacet((host) => { + host.serve( + ChatModeSetting, + ({ conversation }, self) => { + const id = conversation as ConversationId; + const harness = harnessOf(self); + const read = async (ctx: Context) => + (await harness().durable.snapshot(ChatModeDoc, id, durableContext(ctx))) ?? { + enabled: false, + name: DEFAULT_AGENT_NAME, + }; + const shown = self.state(Shown); + const commits = self.open( + async (ctx) => { + const durable = harness().durable; + const doc = await read(ctx); + await shown.change((draft) => { + draft.enabled = doc.enabled; + draft.name = doc.name; + }, ctx); + if (!(await durable.conversation(id, ctx))) throw new Error(`Unknown conversation ${conversation}`); + // The waiting count also follows the entries and the queue. + return new CommitSignal(durable, (publication) => + publication.changes.some( + (change) => + (change.type === "entry" && change.value.conversationId === id) || + (change.type === "document" && + (change.record.kind === ChatModeDoc.definition.kind || + change.record.kind === InboxDoc.definition.kind) && + change.conversationId === id), + ), + ); + }, + (signal) => signal.close(), + ); + // The document is the truth; the cells show it. Subscribed before reading, so no commit is missed. The view + // holds only a window of the entries, so the router's backwards scan counts what waits, while chat mode is on. + self.onLive(async (ctx) => { + const signal = signalOf(ctx); + while (!signal.aborted) { + const seen = commits.value.version; + const doc = await read(ctx); + const durable = harness().durable; + const count = doc.enabled + ? unmentioned( + await recentEntries(durable, id, ctx), + await durable.snapshot(InboxDoc, id, durableContext(ctx)), + ).length + : 0; + const current = shown.value; + if (current.enabled !== doc.enabled || current.name !== doc.name || current.waiting !== count) + await shown.change((draft) => { + draft.enabled = doc.enabled; + draft.name = doc.name; + draft.waiting = count; + }, ctx); + await commits.value.next(seen, signal).catch(() => {}); + } + }); + const commit = async (ctx: Ctx, change: (doc: { enabled: boolean; name: string }) => void) => { + const durable = await harness().durable.conversation(id, durableContext(ctx)); + if (!durable) throw invalid("args", `Unknown conversation ${conversation}`); + await durable.commit(async (tx) => change(await tx.doc(ChatModeDoc, id)), durableContext(ctx)); + }; + return { + enabled: self.derive([shown], ([value]) => value.enabled), + name: self.derive([shown], ([value]) => value.name), + waiting: self.derive([shown], ([value]) => value.waiting), + set: async ({ enabled: on }, ctx) => { + await commit(ctx, (doc) => { + doc.enabled = on; + }); + return null; + }, + rename: async ({ name: next }, ctx) => { + await commit(ctx, (doc) => { + doc.name = next; + }); + return null; + }, + }; + }, + { linger: 30_000 }, + ); + + contribute(host, "p2.chat-mode-router", (self) => { + const harness = harnessOf(self); + /** Validates upload IDs of this session (copying them to a remote environment, like any input's). */ + const resolve = async (ids: readonly string[], conversation: number, ctx: Ctx) => { + const attachments = await ctx.get(Attachments, {}); + const files = []; + for (let start = 0; start < ids.length; start += MAX_ATTACHMENTS) + files.push( + ...(await attachments.resolve({ ids: ids.slice(start, start + MAX_ATTACHMENTS), conversation }, ctx)), + ); + return files; + }; + self.contribute(SendRouters, { + "chat-mode": async ({ conversation, author, text, requestId, whenBusy, attachments, replyTo }, ctx) => { + const durable = harness().durable; + const id = conversation as ConversationId; + const settings = await durable.snapshot(ChatModeDoc, id, durableContext(ctx)); + if (!settings?.enabled) return undefined; + const chat = await ctx.get(Conversation, { conversation }); + // The client names the entry; whom it quotes and what comes from the entry. + const answering = replyTo === undefined ? undefined : await replyTarget(durable, id, replyTo, ctx); + if (!mentionsAgent(text, settings.name ?? DEFAULT_AGENT_NAME)) { + const files = await resolve(attachments, conversation, ctx); + const data: ChatMessage = { + author, + text, + attachments: files.map(({ id, name }) => ({ id, name })), + ...(answering ? { replyTo: answering } : {}), + }; + return chat.write({ kind: CHAT_ENTRY, data, requestId, author }, ctx); + } + const inbox = await durable.snapshot(InboxDoc, id, durableContext(ctx)); + const earlier = unmentioned(await recentEntries(durable, id, ctx), inbox); + const files = await resolve(digestFiles(earlier, attachments), conversation, ctx); + const own = files.filter((file) => attachments.includes(file.id)); + const mention = { + author, + text, + chat: earlier, + replyTo: answering, + attachments: own, + files, + requestId, + whenBusy, + }; + return chat.submit(composeMention(mention), ctx); + }, + }); + }); + + host.serve(ChatMode, ({ session, conversation }, principal, self) => { + if (!canView(principal, session)) throw forbidden("viewers only"); + const setting = self.get(ChatModeSetting, { conversation }); + if (!isOwner(principal)) + return { + variant: "reader", + members: { enabled: setting.enabled, waiting: setting.waiting, name: setting.name }, + }; + return { + variant: "owner", + members: { + enabled: setting.enabled, + waiting: setting.waiting, + name: setting.name, + set: (change, ctx) => setting.set(change, ctx), + rename: (change, ctx) => setting.rename(change, ctx), + }, + }; + }); +}); diff --git a/apps/p2/plugins/chat/contract.ts b/apps/p2/plugins/chat/contract.ts new file mode 100644 index 000000000..3226c213a --- /dev/null +++ b/apps/p2/plugins/chat/contract.ts @@ -0,0 +1,308 @@ +import type { ConversationView } from "@earendil-works/pi-durable"; +import { arr, bool, int, nul, num, obj, opt, type Static, str, union, unknown } from "@earendil-works/pi-schema"; +import type { Author, Ctx } from "@p2/app/v1"; +import { + AttachmentSchema, + EntriesPageSchema, + EntriesQuerySchema, + keyedRegistry, + literal, + local, + MAX_ATTACHMENTS, + ModelRefSchema, + method, + param, + propertiesOf, + remote, + sessionHost, + ThinkingLevelSchema, + unsafe, +} from "@p2/app/v1"; +import type { ActiveStats } from "@p2/app/v1/worker"; + +/** + * Durable's conversation view as clients hold it: a window over the transcript (the newest 50 to 100 entries, `before` + * the entry just before it) and the conversation's documents; null until the watch delivered it. Clients render it + * through `ConversationTranscript`, which pages in what lies before the window or in a gap (`Chat.entries`). + */ +export const ViewSchema = unsafe(unknown()); + +/** Numbers about the conversation's active context, which the window does not hold; null until computed. */ +export const StatsSchema = unsafe(unknown()); + +/** One response's cost, for the cost sheet (`Chat.costs`). */ +export const AssistantCostSchema = obj({ + entryId: str(), + timestamp: num(), + cost: num(), + /** Absent when the response reported no prompt tokens. */ + cacheHit: opt(num()), + provider: str(), + model: str(), + stopReason: str(), +}); + +const AuthorSchema = obj({ + kind: union([literal("owner"), literal("guest")]), + id: str(), + name: str(), +}); + +export const SendSchema = obj({ + text: str({ maxLength: 1_000_000 }), + /** Client-chosen; a retried send with the same ID is admitted once (scoped to the author). */ + requestId: str({ minLength: 1, maxLength: 200 }), + /** While the conversation is busy: steer the current run (default) or queue a follow-up. */ + whenBusy: opt(union([literal("steer"), literal("followUp")])), + /** Upload IDs (POST /upload) of this session. */ + attachments: opt(arr(str(), { maxItems: MAX_ATTACHMENTS })), + /** + * The entry of this conversation the message answers. Only a router uses it (chat-mode, in chat mode); it checks + * the entry and derives whom it quotes. Ordinary inputs ignore it. + */ + replyTo: opt(int({ minimum: 1 })), +}); + +export { + type EntriesPage, + EntriesPageSchema, + EntriesQuerySchema, + MAX_ENTRIES_PAGE, +} from "@p2/app/v1"; + +/** What `Chat.configure` takes: a known thinking level (version 2). */ +export const ConfigureSchema = obj({ + model: opt(ModelRefSchema), + thinkingLevel: opt(ThinkingLevelSchema), +}); +/** The worker-internal form (`Conversation.configure`): Chat validated it already. */ +const ConversationConfigureSchema = obj({ + model: opt(ModelRefSchema), + thinkingLevel: opt(str()), +}); + +const RestoredSchema = arr(obj({ author: AuthorSchema, text: str() })); + +/** + * The conversation's user shell command (`!cmd`/`!!cmd`) while it runs and until its `p2.shell` entry is placed: + * the tail of its output (src/runtime/shell.ts limits). Null when none. + */ +export const ShellStateSchema = union([ + obj({ + id: str(), + command: str(), + author: AuthorSchema, + includeInContext: bool(), + output: str(), + truncated: bool(), + running: bool(), + exitCode: opt(num()), + cancelled: bool(), + error: opt(str()), + }), + nul(), +]); +export type ShellState = Static; +/** pi's text when a second command is started while one runs (interactive-mode `isBashRunning`). */ +export const SHELL_BUSY = "A bash command is already running. Press Esc to cancel it first."; +export const ShellSchema = obj({ + command: str({ minLength: 1, maxLength: 100_000 }), + /** `!` (true): the output joins the agent's context; `!!` (false): display only. */ + includeInContext: bool(), + /** Client-chosen; a retried call with the same ID runs once (scoped to the author). */ + requestId: str({ minLength: 1, maxLength: 200 }), +}); + +/** + * The conversation's current model as anyone who may view it sees it (guest-safe: no credentials, no catalog): what + * a footer needs for the context percentage and a picker for the thinking levels. Null without a model, or when the + * model is not in the server's catalog. + */ +export const ModelMetaSchema = obj({ + provider: str(), + modelId: str(), + name: str(), + contextWindow: num(), + reasoning: bool(), + thinkingLevels: arr(str()), +}); +export type ModelMeta = Static; +const ModelStateSchema = union([ModelMetaSchema, nul()]); + +export const CompactSchema = obj({ + /** Extra instructions for the summary (`/compact `). */ + instructions: opt(str({ maxLength: 10_000 })), +}); +export const ResetSchema = obj({ + /** Text the new context starts with (a handoff note), as a user message. */ + handoff: opt(str({ maxLength: 100_000 })), +}); +/** `dequeueOwn`: the caller's queued texts in submission order, and how many of theirs raced (placed or withdrawn). */ +export const DequeueOwnResultSchema = obj({ + restored: arr(str()), + skipped: int({ minimum: 0 }), +}); + +/** + * An input as `Conversation.submit` takes it: the text the model reads, who wrote it, and its resolved attachments. + * It is placed as its own entry: `entry`, or by default a `p2.input` entry whose data is the author, the text and the + * attachments (see src/runtime/input.ts). The model message also starts with the author's speaker block. + */ +export const SubmitSchema = obj({ + ...propertiesOf(SendSchema), + author: AuthorSchema, + /** The input's own entry kind and data, for a plugin's inputs (chat-mode's `chat.mention`). */ + entry: opt(obj({ kind: str({ minLength: 1 }), data: unknown() })), + /** Resolved attachments: listed by path, images also inline when the model accepts them. */ + files: opt(arr(AttachmentSchema)), + /** Inline at most this many images: the last ones of `files` (default: all that qualify). */ + maxImages: opt(int({ minimum: 0 })), +}); +export type Submit = Static; + +/** + * One Durable conversation of this worker (host-internal, shared by every viewer): its view mirrored from a Durable + * watch, and the operations that need an author, which callers pass explicitly. + */ +export const Conversation = local("p2.conversation", { + version: 1, + params: { conversation: param.int() }, + members: { + /** The view's window, as every viewer holds it. */ + view: ViewSchema, + /** The active context's numbers. */ + stats: StatsSchema, + /** Each response of the active range with its cost. */ + costs: method(undefined, arr(AssistantCostSchema)), + submit: method(SubmitSchema, obj({ submissionId: num() })), + /** + * Appends a display-only entry (no model message) through a Durable write submission: placed at once when idle, + * at the run's next boundary when busy. A retried `requestId` is admitted once per author. + */ + write: method( + obj({ + kind: str({ minLength: 1 }), + data: unknown(), + requestId: str({ minLength: 1, maxLength: 200 }), + author: AuthorSchema, + }), + obj({ submissionId: num() }), + ), + stop: method(undefined, obj({ restored: RestoredSchema })), + dequeue: method(obj({ submissionId: num(), restore: bool() }), obj({ restored: RestoredSchema })), + entries: method(EntriesQuerySchema, EntriesPageSchema), + configure: method(ConversationConfigureSchema), + /** The current model's metadata (see `ModelMetaSchema`), following the view's `pi.agent` and the catalog. */ + model: ModelStateSchema, + /** Admits a manual compaction task (Durable `Conversation.compact`). */ + compact: method(CompactSchema, obj({ taskId: num() })), + /** Starts a new context (Durable `Conversation.reset`), placed at the next boundary while busy. */ + reset: method(ResetSchema), + /** Aborts this conversation's live compaction tasks; returns how many were marked. */ + abortCompaction: method(undefined, obj({ aborted: int({ minimum: 0 }) })), + /** Withdraws `authorId`'s queued inputs (by the server-stamped author) in submission order. */ + dequeueOwn: method(obj({ authorId: str() }), obj({ restored: RestoredSchema, skipped: int({ minimum: 0 }) })), + /** The running (or just finished, not yet placed) user shell command. */ + shell: ShellStateSchema, + /** Runs a user shell command on the conversation's environment; one at a time per conversation. */ + runShell: method(obj({ ...propertiesOf(ShellSchema), author: AuthorSchema }), obj({ shellId: str() })), + /** Cancels the running command `shellId`; false when it is not the running one. */ + abortShell: method(obj({ shellId: str() }), obj({ aborted: bool() })), + }, +}); + +/** + * A conversation as a principal sees it: viewers read, participants also send, stop and configure (owner), compact, + * abort compaction, restore their own queued inputs and (owner) reset the context. + */ +export const Chat = remote("p2.chat", { + /** + * 2: `configure` takes only known thinking levels; `send` refuses blank text without attachments. + * 3: `model` state; `compact`, `abortCompaction`, `reset` and `dequeueOwn`. + * 4: `liveShell` state; `shell` and `abortShell` (user shell commands, `!cmd`/`!!cmd`). + * 5: `view` holds the newest entries only (`ChatView.window`; older ones through `history`). + * 6: `view` is Durable's window over the transcript (`before`); `stats` state; `entries` replaces `history`. + */ + version: 6, + params: { session: param.str(), conversation: param.int() }, + hostAddress: sessionHost, + variants: { + viewer: { + state: { view: ViewSchema, stats: StatsSchema, model: ModelStateSchema, liveShell: ShellStateSchema }, + methods: { entries: method(EntriesQuerySchema, EntriesPageSchema) }, + }, + participant: { + state: { view: ViewSchema, stats: StatsSchema, model: ModelStateSchema, liveShell: ShellStateSchema }, + methods: { + entries: method(EntriesQuerySchema, EntriesPageSchema), + send: method(SendSchema, obj({ submissionId: num() })), + /** Aborts the current run; queued inputs are withdrawn and the caller's own come back as text. */ + stop: method(undefined, obj({ restored: arr(str()) })), + /** Withdraws a queued input; with `restore`, returns its text if the caller wrote it. */ + dequeue: method(obj({ submissionId: num(), restore: opt(bool()) }), obj({ text: union([str(), nul()]) })), + configure: method(ConfigureSchema), + /** Summarizes earlier context now (pi's `/compact [instructions]`); returns the compaction task. */ + compact: method(CompactSchema, obj({ taskId: num() })), + /** Aborts running compactions (pi: Escape while compacting); returns how many were marked. */ + abortCompaction: method(undefined, obj({ aborted: int({ minimum: 0 }) })), + /** Starts a new context with an optional handoff (owner only: it changes the context for everyone). */ + reset: method(ResetSchema), + /** + * Withdraws the caller's own queued inputs (by the server-stamped author, in submission order) and returns + * their texts; other authors' stay queued. Inputs placed or withdrawn concurrently are counted as skipped. + */ + dequeueOwn: method(undefined, DequeueOwnResultSchema), + /** + * Runs a shell command in this conversation's working directory on the session's environment (pi's `!cmd`; + * `!!cmd` without context). It never starts a run: the finished command becomes a `p2.shell` entry. One + * command at a time per conversation. + */ + shell: method(ShellSchema, obj({ shellId: str() })), + /** Cancels the running command (its author or the owner). */ + abortShell: method(obj({ shellId: str() }), obj({ aborted: bool() })), + }, + }, + }, +}); + +/** + * Each response of a conversation's active range with its cost, for whoever may view it (the cost sheet: a client's + * view holds only a window). + */ +export const ChatCosts = remote("p2.chat-costs", { + version: 1, + params: { session: param.str(), conversation: param.int() }, + hostAddress: sessionHost, + variants: { + viewer: { methods: { costs: method(undefined, arr(AssistantCostSchema)) } }, + }, +}); + +/** What a participant sent, with the server-stamped author, as `Chat.send` hands it to a router. */ +export type RoutedSend = Omit, "attachments"> & { + readonly conversation: number; + readonly author: Author; + readonly attachments: readonly string[]; +}; + +/** + * Worker extension point of `Chat.send`: each router may take a message (returning its submission) or leave it + * (`undefined`). Untaken messages become ordinary inputs. Example: chat-mode writes chat entries. + */ +export const SendRouters = + keyedRegistry<(input: RoutedSend, ctx: Ctx) => Promise<{ submissionId: number } | undefined>>( + "p2.chat-send-routers", + ); + +/** What a plugin adds to the display of its entry kind's messages: the message it answers, and a short note. */ +export type EntryDisplay = { + readonly replyTo?: { readonly entryId: number; readonly name: string; readonly excerpt: string }; + readonly note?: string; +}; + +/** + * Browser extension point of the transcript, keyed by entry kind (e.g. `chat.mention`). The transcript shows every + * entry whose data is a person's message by itself; a registered kind adds to it. + */ +export const TranscriptEntries = + keyedRegistry<(data: unknown) => EntryDisplay | undefined>("p2.chat-transcript-entries"); diff --git a/apps/p2/plugins/chat/outbox.ts b/apps/p2/plugins/chat/outbox.ts new file mode 100644 index 000000000..a65572833 --- /dev/null +++ b/apps/p2/plugins/chat/outbox.ts @@ -0,0 +1,167 @@ +/** + * Messages sent from the composer, until the server has them: sending never waits for the connection. A message shows + * at once as it will look (`Outgoing.item`), and the outbox sends its messages in order whenever a sender is connected + * (the conversation's view is open and its chat object live), each with its own request ID: a retried send is admitted + * once, so a send whose answer was lost to a broken connection is simply sent again. A message the server refuses goes + * back to the composer (`Sender.restore`); one it accepted stays shown until the transcript shows it (`shown`). + * + * One outbox per conversation, kept for the page's lifetime: switching conversations keeps unsent messages, and they + * go out once their conversation is open again. + */ +import type { Item, PickedFile, ReplyTarget } from "@p2/app/v1/web"; + +export type Outgoing = { + /** The request ID: a retry of this message is admitted once. */ + readonly id: string; + readonly text: string; + readonly files: readonly PickedFile[]; + readonly whenBusy: "steer" | "followUp"; + readonly replyTo?: ReplyTarget; + /** The message as the transcript will show it (a person's message), shown until it does. */ + readonly item: Extract; + /** Accepted: its submission (a queued input's ID), and the conversation's newest entry before it. */ + readonly sent?: { readonly submissionId: number; readonly after: number }; +}; + +/** How an open conversation sends: the server's call, and what to do with a refused message. */ +export type Sender = { + /** Sends `message` (uploads, expansion, the call); resolves with what the server accepted. */ + send(message: Outgoing): Promise>; + /** Whether `error` refuses the message for good (not allowed, invalid); anything else is retried. */ + refused(error: unknown): boolean; + /** Puts a refused message back into the composer, saying why. */ + restore(message: Outgoing, error: unknown): void; +}; + +/** How long an accepted message stays shown when the transcript does not show it (it may never: a dropped steer). */ +const SHOWN_AT_MOST = 10_000; +const RETRY_MIN = 500; +const RETRY_MAX = 10_000; +/** + * Failed tries while connected after which a message goes back to the composer (about half a minute): an error that + * keeps coming back is not the connection. A reconnect starts the count again. + */ +const GIVE_UP = 6; + +export class Outbox { + readonly #retryDelay: (attempt: number) => number; + readonly #shownAtMost: number; + #messages: readonly Outgoing[] = []; + readonly #listeners = new Set<() => void>(); + #sender: Sender | undefined; + #running = false; + /** The message being sent now: the server may already have it. */ + #inFlight: string | undefined; + #attempt = 0; + #retry: ReturnType | undefined; + + /** Timings: the wait before retry `attempt` (from 0), and how long an accepted message stays shown at most. */ + constructor(timing: { retryDelay?: (attempt: number) => number; shownAtMost?: number } = {}) { + this.#retryDelay = timing.retryDelay ?? ((attempt) => Math.min(RETRY_MAX, RETRY_MIN * 2 ** attempt)); + this.#shownAtMost = timing.shownAtMost ?? SHOWN_AT_MOST; + } + + subscribe = (listener: () => void): (() => void) => { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + }; + snapshot = (): readonly Outgoing[] => this.#messages; + + /** Queues `message` and sends it as soon as a sender is connected. */ + add(message: Outgoing): void { + this.#set([...this.#messages, message]); + void this.#run(); + } + + /** Takes back `id` if it is not sent (nor being sent): the queued chip's "move back to editor". */ + take(id: string): Outgoing | undefined { + const message = this.#messages.find((candidate) => candidate.id === id); + if (!message || message.sent || message.id === this.#inFlight) return undefined; + this.remove(id); + return message; + } + + /** Drops `id`, sent or not. */ + remove(id: string): void { + this.#set(this.#messages.filter((message) => message.id !== id)); + } + + /** + * Takes back the messages not sent yet (Stop puts them into the composer, as the server's queued inputs). Not the + * one being sent: the server may have it already, and Stop's own answer returns it if so. + */ + takeUnsent(): Outgoing[] { + const taken = (message: Outgoing) => !message.sent && message.id !== this.#inFlight; + const unsent = this.#messages.filter(taken); + if (unsent.length) this.#set(this.#messages.filter((message) => !taken(message))); + return unsent; + } + + /** How to send now, or undefined while sending cannot work (offline, the chat object not live). */ + connect(sender: Sender | undefined): void { + this.#sender = sender; + if (sender) { + this.#attempt = 0; + void this.#run(); + } + } + + /** Drops accepted messages that `shown` says the transcript shows now. */ + settle(shown: (message: Outgoing) => boolean): void { + const kept = this.#messages.filter((message) => !message.sent || !shown(message)); + if (kept.length !== this.#messages.length) this.#set(kept); + } + + #set(messages: readonly Outgoing[]): void { + this.#messages = messages; + for (const listener of [...this.#listeners]) listener(); + } + + /** Sends the unsent messages in order, one at a time; a failure that is not a refusal waits and tries again. */ + async #run(): Promise { + if (this.#running) return; + this.#running = true; + clearTimeout(this.#retry); + try { + for (;;) { + const sender = this.#sender; + const next = this.#messages.find((message) => !message.sent); + if (!sender || !next) return; + this.#inFlight = next.id; + try { + const sent = await sender.send(next); + this.#attempt = 0; + // Removed meanwhile (its chip's ×): the server has it now and shows it. + if (this.#messages.some((message) => message.id === next.id)) + this.#set(this.#messages.map((message) => (message.id === next.id ? { ...message, sent } : message))); + setTimeout(() => this.remove(next.id), this.#shownAtMost); + } catch (error) { + if (sender.refused(error) || this.#attempt + 1 >= GIVE_UP) { + this.#attempt = 0; + this.remove(next.id); + sender.restore(next, error); + continue; + } + this.#retry = setTimeout(() => void this.#run(), this.#retryDelay(this.#attempt++)); + return; + } finally { + this.#inFlight = undefined; + } + } + } finally { + this.#running = false; + } + } +} + +const outboxes = new Map(); + +/** The outbox of conversation `key` (`session#conversation`). */ +export function outboxOf(key: string): Outbox { + let outbox = outboxes.get(key); + if (!outbox) { + outbox = new Outbox(); + outboxes.set(key, outbox); + } + return outbox; +} diff --git a/apps/p2/plugins/chat/package.json b/apps/p2/plugins/chat/package.json new file mode 100644 index 000000000..6e09afc1d --- /dev/null +++ b/apps/p2/plugins/chat/package.json @@ -0,0 +1,5 @@ +{ + "name": "chat", + "private": true, + "type": "module" +} diff --git a/apps/p2/plugins/chat/tui-commands.ts b/apps/p2/plugins/chat/tui-commands.ts new file mode 100644 index 000000000..3f2795994 --- /dev/null +++ b/apps/p2/plugins/chat/tui-commands.ts @@ -0,0 +1,169 @@ +// Chat's TUI commands (p2-tui-port.md section 6.3), ported from the p2.* actions to pi's ids. They reach the live +// surface through `ctx.surface("chat")`, so a facet reload never leaves a stale handler. +import type { TuiCommand, TuiContext } from "@p2/app/v1/tui"; +import { ChatSurface } from "./tui.ts"; + +const chat = (ctx: TuiContext): ChatSurface | undefined => { + const surface = ctx.surface("chat"); + return surface instanceof ChatSurface ? surface : undefined; +}; + +export const CHAT_COMMANDS: Record = { + "app.interrupt": { + title: "Stop the agent", + category: "Chat", + // pi: Escape closes the editor's autocomplete first, and aborts a running compaction. + when: "chat && participant && !autocomplete && !overlay", + run: (ctx) => chat(ctx)?.interrupt(), + }, + "app.clear": { + title: "Clear editor", + category: "Chat", + // pi: Ctrl+C clears the editor; a second press within 500 ms exits (ChatSurface.clear). + when: "chat && editorFocused && !autocomplete && !overlay", + run: (ctx) => chat(ctx)?.clear(), + }, + "app.message.followUp": { + title: "Queue follow-up message", + category: "Chat", + when: "chat && participant && !overlay", + run: (ctx) => chat(ctx)?.send("followUp"), + }, + "app.message.dequeue": { + title: "Restore your queued messages", + category: "Chat", + when: "chat && participant && !overlay", + run: (ctx) => chat(ctx)?.dequeueOwn(), + }, + "p2.chat.compact": { + title: "Compact the context", + category: "Session", + when: "chat && participant && !overlay", + slash: { name: "compact", argumentHint: "[instructions]" }, + run: (ctx, args) => chat(ctx)?.compact(args), + }, + // pi's /fork and /clone as new sessions (Sessions.fork/clone; the server copies the store). + "app.session.fork": { + title: "Fork from a message into a new session", + category: "Session", + when: "chat && owner && !overlay", + slash: { name: "fork" }, + run: (ctx) => chat(ctx)?.openFork(), + }, + "p2.session.clone": { + title: "Clone the session", + category: "Session", + when: "chat && owner && !overlay", + slash: { name: "clone" }, + run: (ctx) => chat(ctx)?.cloneSession(), + }, + "p2.chat.newContext": { + title: "Start a new context", + category: "Session", + when: "chat && participant && owner && !overlay", + slash: { name: "new-context", argumentHint: "[handoff]" }, + run: (ctx, args) => chat(ctx)?.newContext(args), + }, + // P9: local files reach the server only through the launcher (its picker or a confirmation of the path). + "p2.chat.attach": { + title: "Attach a local file", + category: "Chat", + when: "chat && participant && !overlay", + slash: { name: "attach", argumentHint: "[path]" }, + run: async (ctx, args) => { + const result = await ctx.grant("attach", args ? { path: args } : {}); + const file = result.ok ? parseUpload(result.value) : undefined; + if (file) chat(ctx)?.addAttachment(file); + else if (!result.ok && result.error !== "denied") ctx.notify(`Not attached (${result.error})`, "warning"); + }, + }, + "p2.chat.detach": { + title: "Remove the last attachment", + category: "Chat", + keys: "backspace", + when: "chat && participant && editorEmpty && attachments && !overlay", + run: (ctx) => chat(ctx)?.detach(), + }, + "p2.chat.detachAll": { + title: "Remove all attachments", + category: "Chat", + when: "chat && participant && attachments && !overlay", + slash: { name: "detach" }, + run: (ctx) => chat(ctx)?.detach(true), + }, + // pi's /copy (P5): the launcher shows the full text and copies only after a local confirmation (C grant). + "p2.chat.copy": { + title: "Copy the last answer", + category: "Chat", + when: "chat && !overlay", + slash: { name: "copy" }, + run: async (ctx) => { + const text = chat(ctx)?.lastAssistantText(); + if (!text) { + ctx.notify("Nothing to copy yet", "warning"); + return; + } + const result = await ctx.grant("copy", { text: text.slice(0, 1024 * 1024), label: "the last answer" }); + if (!result.ok && result.error !== "denied") ctx.notify(`Not copied (${result.error})`, "warning"); + }, + }, +}; + +/** An upload the launcher made (P9): the server's upload ID and what the chip shows. Never a local path. */ +export type LauncherUpload = { id: string; name: string; size: number; mimeType?: string }; +export function parseUpload(value: unknown): LauncherUpload | undefined { + if (!value || typeof value !== "object") return undefined; + const { id, name, size, mimeType } = value as Record; + if (typeof id !== "string" || !/^[0-9a-f-]{36}$/i.test(id) || typeof name !== "string" || typeof size !== "number") + return undefined; + return { + id, + name: name.slice(0, 255), + size, + ...(typeof mimeType === "string" ? { mimeType: mimeType.slice(0, 100) } : {}), + }; +} +export const formatSize = (bytes: number): string => + bytes < 1024 + ? `${bytes} B` + : bytes < 1048576 + ? `${Math.round(bytes / 1024)} KB` + : `${(bytes / 1048576).toFixed(1)} MB`; + +/** + * Answers to the launcher's grant-bound questions (section 3.1 rule 4, `externalEditor`): the text Ctrl+X copies, and + * the editor text for Ctrl+G and its result. Only the mounted chat surface answers; the launcher drops results of a + * grant that ended (navigation, swap). + */ +export function registerLauncherAnswers(surface: ChatSurface): () => void { + const answers = surface.runtime.launcherAnswers; + const own: [string, (payload: unknown) => unknown][] = [ + ["copyText", () => ({ text: surface.lastAssistantText() ?? "" })], + // Ctrl+V and p2.attach: the launcher uploaded a local file after its own confirmation. + [ + "attached", + (payload) => { + const file = parseUpload(payload); + return { added: file ? surface.addAttachment(file) : false }; + }, + ], + ["editorText", () => (surface.editable ? { text: surface.editor.getExpandedText() } : null)], + [ + "editorResult", + (payload) => { + const text = + payload && typeof payload === "object" && "text" in payload && typeof payload.text === "string" + ? payload.text + : undefined; + if (text === undefined || !surface.editable) return { applied: false }; + surface.editor.setText(text); + surface.runtime.changed(); + return { applied: true }; + }, + ], + ]; + for (const [kind, answer] of own) answers.set(kind, answer); + return () => { + for (const [kind, answer] of own) if (answers.get(kind) === answer) answers.delete(kind); + }; +} diff --git a/apps/p2/plugins/chat/tui.ts b/apps/p2/plugins/chat/tui.ts new file mode 100644 index 000000000..2b7ab92f7 --- /dev/null +++ b/apps/p2/plugins/chat/tui.ts @@ -0,0 +1,981 @@ +// The chat screen: coding-agent's fullscreen chat layout (chat-viewport.ts, DurableTui) over a p2 conversation. +// The transcript is incremental components in a ScrollView (pi-tui's scroll and search keys), the dock holds the +// pending area and pi's editor with the working indicator in its border. Everything the reader leaves behind is kept +// per viewed conversation, (session, conversation): references, drafts, pending send IDs and the reading position (P0c). + +import { randomUUID } from "node:crypto"; +import type { ConversationView, LiveState } from "@earendil-works/pi-durable"; +import { + type Component, + Container, + ScrollView, + Text, + TruncatedText, + truncateToWidth, + VStack, +} from "@earendil-works/pi-tui"; +import type { Held, HeldIn } from "@p2/app/v1"; +import { BACKGROUND_CONTEXT, FileIndex, followRegistry, liveIn, serveKeyed, statusOf } from "@p2/app/v1"; +import { + BashExecutionComponent, + buildTranscript, + CHAT_SURFACE, + type ChatSurfaceApi, + CompactionStatusIndicator, + CustomEditor, + type EntryDisplayOf, + FileCompletionProvider, + type FileSource, + type Gap, + gapKey, + identityKey, + lineText, + type MarkdownTransformer, + ordered, + type Pages, + pagesWatchable, + type ReplyTarget, + RetryStatusIndicator, + resolveToolRenderers, + type StatusIndicator, + shellAuthor, + type ToolRendererResolver, + type ToolRenderers, + type ToolView, + type Transcript, + type TranscriptContext, + type TranscriptStore, + TranscriptView, + type TuiEntryRenderer, + TuiEntryRenderers, + TuiMarkdownTransformers, + type TuiRuntime, + TuiScope, + type TuiSurface, + TuiToolRendererResolvers, + TuiToolRenderers, + transcriptOf, + tuiFacet, + UserMessageSelectorComponent, + WorkingStatusIndicator, +} from "@p2/app/v1/tui"; +import { Drafts } from "../drafts/contract.ts"; +import { MAX_ATTACHMENTS } from "../files/contract.ts"; +import { type Session, Sessions } from "../sessions/contract.ts"; +import { Chat, type EntryDisplay, SHELL_BUSY, type ShellState, TranscriptEntries } from "./contract.ts"; +import { CHAT_COMMANDS, formatSize, type LauncherUpload, registerLauncherAnswers } from "./tui-commands.ts"; + +/** pi: a second Ctrl+C within this window exits; a second Escape runs `doubleEscapeAction`. */ +const DOUBLE_CTRL_C_MS = 500; +const DOUBLE_ESCAPE_MS = 500; + +export class ChatSurface implements TuiSurface, ChatSurfaceApi { + readonly [CHAT_SURFACE] = true as const; + readonly runtime: TuiRuntime; + readonly editor: CustomEditor; + focus: CustomEditor; + readonly transcript: TranscriptView; + readonly scroll: ScrollView; + private readonly pending = new Container(); + private readonly status = new Container(); + private readonly input = new Container(); + private readonly dock: VStack; + /** The composer bar's chips (`composer:` slots, the web's SlotList "composer"), one line under the editor. */ + private readonly chips: Component = { + render: (width) => { + const theme = this.runtime.theme.current; + const files = (this.identity ? this.attachments.get(this.identity) : undefined) ?? []; + const parts = [ + ...files.map((file) => theme.fg("accent", `[${lineText(file.name)} ${formatSize(file.size)}]`)), + ...this.runtime.slots("composer:").flatMap((surface) => surface.render(width)), + ]; + return parts.length ? [truncateToWidth(` ${parts.join(" ")}`, width)] : []; + }, + invalidate: () => {}, + }; + /** Replies exist while a plugin allows them (chat-mode, in chat mode), as the web's `canReply`. */ + canReply = false; + /** The agent's name to mention while chat mode is on (chat-mode's setting), offered first as `@name`. */ + mentionName = "Pi"; + /** The message the next send answers. */ + replyTo: { entryId: number; name: string } | undefined; + private readonly readOnly = new Text("", 1, 0); + private scope: TuiScope | undefined; + private sessions: TuiScope; + /** The viewed identity, `identityKey(session, conversation)`. */ + private identity: string | undefined; + private session: string | undefined; + private conversation = 1; + private rows: readonly Session[] = []; + /** Where the viewed conversation works (its agent's cwd: a subagent's own, else the session's). */ + private get cwd(): string { + const agent = this.view?.docs["pi.agent"] as { cwd?: unknown } | undefined; + return typeof agent?.cwd === "string" ? agent.cwd : (this.rows.find((row) => row.id === this.session)?.cwd ?? ""); + } + private ref: (() => Held) | undefined; + private view: ConversationView | null | undefined; + /** The conversation's user shell command while it runs (`Chat.liveShell`), and its component (the transcript tail). */ + private liveShell: ShellState = null; + private shellView: { id: string; component: BashExecutionComponent } | undefined; + /** + * The conversation's transcript (src/web/history.ts): entries that leave the view's window stay. Kept with the + * lingering Chat reference, so switching back to a conversation finds its pages loaded. + */ + private transcriptStore: TranscriptStore | undefined; + private get pages(): Pages | undefined { + return this.transcriptStore?.value; + } + /** Transcripts whose earlier page failed to load: reaching the top again does not ask again. */ + private readonly earlierFailed = new WeakSet(); + /** Gaps whose entries failed to load: showing them again does not ask again. */ + private readonly gapsFailed = new WeakSet(); + private current: Transcript | undefined; + private synced: unknown[] = []; + private disposed = false; + private sending = false; + /** A send asked for while the conversation was connecting. */ + private waiting: "steer" | "followUp" | undefined; + private lastCtrlC = 0; + private lastEscape = 0; + private indicator: StatusIndicator | undefined; + private indicatorKey = ""; + private toolRenderers: ReadonlyMap = new Map(); + private resolvers: ReadonlyMap = new Map(); + private entryRenderers: ReadonlyMap = new Map(); + private transformers: MarkdownTransformer[] = []; + private displays: ReadonlyMap EntryDisplay | undefined> = new Map(); + private readonly display: EntryDisplayOf = (kind, data) => this.displays.get(kind)?.(data); + private stops: (() => void)[] = []; + + constructor(runtime: TuiRuntime) { + this.runtime = runtime; + const surface = this; + const ctx: TranscriptContext = { + get theme() { + return runtime.theme; + }, + keyText: (id) => runtime.ctx.keyText(id), + requestRender: () => runtime.tui.requestRender(), + get cwd() { + return surface.cwd; + }, + get markdownTransformers() { + return surface.transformers; + }, + get settings() { + return runtime.ctx.settings; + }, + toolRenderers: (name) => resolveToolRenderers(name, surface.resolvers, surface.toolRenderers), + get entryRenderers() { + return surface.entryRenderers; + }, + }; + this.transcript = new TranscriptView( + ctx, + { expanded: runtime.expandedTools, showThinking: runtime.showThinking }, + { render: (width) => runtime.header.render(width), invalidate: () => runtime.header.invalidate() }, + ); + this.scroll = new ScrollView(this.transcript, { + follow: "end", + primary: true, + overscroll: "chain", + scrollbar: "auto", + scrollbarTrackStyle: (text) => runtime.theme.current.fg("scrollbarTrack", text), + scrollbarThumbStyle: (text) => runtime.theme.current.fg("scrollbarThumb", text), + }); + this.transcript.attach(this.scroll); + this.transcript.onTop = () => this.loadEarlier(); + this.transcript.onVisibleGap = (item) => this.fillGap(item.gap); + this.transcript.onMissingAnchor = (anchor) => { + // A position in older history: page towards it while there is more. + const items = this.pages?.view.items ?? []; + const oldest = items.find((item) => item.kind === "entry"); + const older = oldest === undefined || (anchor.entryId ?? 0) < oldest.entry.id; + if (anchor.entryId === undefined || items[0]?.kind !== "earlier" || !older) + return this.pages?.loadingEarlier === true; + this.loadEarlier(); + return true; + }; + this.editor = new CustomEditor(runtime.tui, runtime.ctx, { paddingX: 1 }); + this.focus = this.editor; + this.editor.onChange = () => { + if (this.identity) runtime.drafts.set(this.identity, this.editor.getExpandedText()); + this.syncBorder(); + }; + this.editor.setAutocompleteProvider( + new FileCompletionProvider( + runtime.ctx.commands.autocomplete, + () => this.fileSource(), + (text) => { + if (runtime.notices.at(-1)?.text !== text) runtime.notify(text); + }, + // Chat mode: `@pi` (or the agent's name there) mentions the agent; offered first while chat mode is on. + () => (this.canReply ? this.mentionName : undefined), + ), + ); + // Editor clears before onSubmit: restore until admission succeeds, including failed/offline sends. + this.editor.onSubmit = (text) => { + // pi's bash mode: `!cmd` runs a command whose output joins the context, `!!cmd` one that stays out of it. + const bang = /^\s*(!!?)\s*([\s\S]*)$/.exec(text); + if (bang?.[2]?.trim()) { + this.editor.setText(text); + this.shell(bang[2].trim(), bang[1] === "!"); + return; + } + if (text.trimStart().startsWith("/")) { + if (runtime.ctx.commands.runSlash(text)) return; + // A known command that does not apply here (not connected yet, a guest, ...) is not sent as a message. + const name = /^\/(\S+)/.exec(text.trimStart())?.[1]; + if (name && runtime.ctx.commands.list().some((item) => item.slash === name)) { + this.editor.setText(text); + runtime.notify(`/${name} is not available here`, "warning"); + return; + } + // A prompt template or /skill: (10.8) is expanded on the session worker, then sent as pi does. + if (name && runtime.ctx.commands.expandable(name)) { + this.editor.setText(text); + void runtime.ctx.commands.expand(text.trimStart()).then((expanded) => { + if (this.editor.getText() === text) this.editor.setText(expanded); + this.send("steer"); + }); + return; + } + } + this.editor.setText(text); + this.send("steer"); + }; + this.input.addChild(this.editor); + this.dock = new VStack([ + { component: this.pending, shrink: 1, minSize: 0 }, + { component: this.status, shrink: 1, minSize: 0 }, + { component: this.input, shrink: 1, minSize: 1 }, + { component: this.chips, shrink: 1, minSize: 0 }, + ]); + const registries = runtime.host; + this.stops.push( + followRegistry(registries, TuiToolRenderers, (table) => { + this.toolRenderers = table; + this.renderersChanged(); + }), + followRegistry(registries, TuiToolRendererResolvers, (table) => { + this.resolvers = table; + this.renderersChanged(); + }), + followRegistry(registries, TuiEntryRenderers, (table) => { + this.entryRenderers = table; + this.renderersChanged(); + }), + followRegistry(registries, TuiMarkdownTransformers, (table) => { + this.transformers = ordered(table); + this.renderersChanged(); + }), + followRegistry(registries, TranscriptEntries, (table) => { + this.displays = table; + this.renderersChanged(); + }), + ); + // The session's cwd: tool paths display relative to it. + this.sessions = new TuiScope(runtime); + this.sessions.watch(Sessions, {}, (ref, follow) => { + const viewer = liveIn(ref, "owner", "guest"); + if (viewer) + follow(viewer.sessions, (rows) => { + this.rows = rows; + }); + }); + const changed = (): void => this.update(); + runtime.listeners.add(changed); + this.stops.push(() => runtime.listeners.delete(changed)); + // What the launcher may ask for under a grant (P5: Ctrl+X copy, Ctrl+G external editor). + this.stops.push(registerLauncherAnswers(this)); + this.update(); + } + + /** Renderer contributions changed (facet reload): rebuild every component with the new tables. */ + private renderersChanged(): void { + this.transcript.reset(); + this.synced = []; + this.sync(); + this.runtime.tui.requestRender(); + } + + layout(): { body: Component; dock: Component } { + return { body: this.scroll, dock: this.dock }; + } + + /** Keeps the reading position of the viewed conversation in the runtime (identity switches and reloads). */ + saveState(): void { + if (!this.identity) return; + const anchor = this.transcript.anchor(); + if (anchor) this.runtime.anchors.set(this.identity, anchor); + else this.runtime.anchors.delete(this.identity); + } + + /** Runtime state changed: switch identity if needed, then sync transcript, dock and indicator. */ + private update(): void { + const runtime = this.runtime; + const identity = runtime.identity; + if (identity !== this.identity) this.switchTo(identity); + this.transcript.setState({ expanded: runtime.expandedTools, showThinking: runtime.showThinking }); + this.sync(); + if (this.waiting && runtime.connected && this.participant()) this.send(this.waiting); + } + + private switchTo(identity: string | undefined): void { + const runtime = this.runtime; + this.saveState(); + this.scope?.dispose(); + this.scope = undefined; + this.ref = undefined; + this.view = undefined; + this.transcriptStore = undefined; + this.files = undefined; + this.current = undefined; + this.liveShell = null; + this.synced = []; + this.sending = false; + this.waiting = undefined; + this.replyTo = undefined; + this.transcript.reset(); + this.identity = identity; + this.session = runtime.session; + this.conversation = runtime.conversation; + this.editor.setText(identity ? (runtime.drafts.get(identity) ?? "") : ""); + this.transcript.restore(identity ? runtime.anchors.get(identity) : undefined); + if (!identity || !this.session) return; + const session = this.session; + const conversation = this.conversation; + const scope = new TuiScope(runtime); + this.scope = scope; + this.ref = scope.watch(Chat, { session, conversation }, (ref, follow) => { + follow(ref.view, (view) => { + if (view !== undefined) this.view = view; + }); + follow(ref.liveShell, (shell) => { + this.liveShell = shell ?? null; + }); + // Every view is merged into the transcript; it is released with the reference. + const store = transcriptOf(runtime.host, identityKey(session, conversation), { + view: ref.view, + entries: (query) => ref.entries(query, BACKGROUND_CONTEXT), + }); + this.transcriptStore = store; + follow(pagesWatchable(store), () => {}); + if (ref.variant !== "participant") return; + scope.watch(Drafts, { session, conversation }, (drafts, subscribe) => { + subscribe(drafts.drafts, (items) => { + if (items?.length) + runtime.perform( + drafts + .claim({ ids: items.map((item) => item.id) }, BACKGROUND_CONTEXT) + .then(({ texts }) => this.restore(identityKey(session, conversation), texts)), + ); + }); + }); + // `@` and Tab file completion (participants, as the workspace); a server without the plugin has none. + if (!this.files) + try { + this.files = scope.watch(FileIndex, { session, conversation }, () => {}); + } catch {} + }); + } + + /** The viewed conversation's file index once live (P7b). */ + private files: (() => Held) | undefined; + private fileSource(): FileSource | undefined { + const ref = liveIn(this.files?.(), "reader"); + return ( + ref && { + complete: (args) => ref.complete(args, BACKGROUND_CONTEXT), + list: (args) => ref.list(args, BACKGROUND_CONTEXT), + } + ); + } + + /** Finished calls of a tool in the loaded transcript, oldest first (`/downloads` lists send_file's). */ + toolViews(name: string): ToolView[] { + return (this.current?.items ?? []).flatMap((item) => + (item.type === "assistant" ? item.tools : item.type === "tool" ? [item.tool] : []).filter( + (tool) => tool.name === name && tool.status === "done", + ), + ); + } + + /** The latest answer's text (pi's /copy and Ctrl+X): the last assistant message's text parts. */ + /** Uploaded attachments (P9) waiting for the next send, by identity; added only by the launcher's uploads. */ + readonly attachments = new Map(); + addAttachment(file: LauncherUpload): boolean { + if (!this.identity || !this.participant()) return false; + const list = this.attachments.get(this.identity) ?? []; + if (list.length >= MAX_ATTACHMENTS || list.some((entry) => entry.id === file.id)) return false; + this.attachments.set(this.identity, [...list, file]); + this.runtime.changed(); + return true; + } + /** Removes the last attachment, or all of them. */ + detach(all = false): void { + if (!this.identity) return; + const list = this.attachments.get(this.identity) ?? []; + this.attachments.set(this.identity, all ? [] : list.slice(0, -1)); + this.runtime.changed(); + } + lastAssistantText(): string | undefined { + const item = this.current?.items.findLast((entry) => entry.type === "assistant" && !entry.streaming); + if (item?.type !== "assistant") return undefined; + const text = item.message.content + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join("\n\n") + .trim(); + return text || undefined; + } + /** Whether this visitor can edit the editor here (a live participant). */ + get editable(): boolean { + return this.participant() !== undefined; + } + + private participant(): HeldIn | undefined { + const ref = this.ref?.(); + return liveIn(ref, "participant"); + } + + private sync(): void { + const ref = this.ref?.(); + const view = this.view; + const pages = this.pages; + const inputs = [pages, this.displays, this.toolRenderers, this.resolvers, this.entryRenderers]; + if (inputs.length !== this.synced.length || inputs.some((input, index) => input !== this.synced[index])) { + this.synced = inputs; + this.current = pages ? buildTranscript(pages.view, this.display, pages.filling) : undefined; + const entries = (pages?.view.items ?? []).flatMap((item) => (item.kind === "entry" ? [item.entry] : [])); + this.transcript.sync(this.current, entries); + } + this.transcript.empty = !this.session + ? "No session selected." + : view === undefined + ? "Loading conversation…" + : view === null + ? "This conversation is not available." + : ""; + const theme = this.runtime.theme; + const participant = this.participant() !== undefined; + this.syncBorder(); + this.syncShell(); + this.syncPending(participant); + this.syncIndicator(participant); + this.input.clear(); + if (participant) this.input.addChild(this.editor); + else { + this.readOnly.setText( + theme.current.fg( + "muted", + !this.session + ? `${this.runtime.ctx.keyText("p2.sessions") || "/sessions"} to choose a session` + : ref !== undefined && statusOf(ref) === "live" + ? // The web's read-only bar: a view-only guest sees under which name it joined. + this.runtime.role === "guest" && this.runtime.grant + ? `Joined as ${lineText(this.runtime.grant.name).slice(0, 40)} (unverified) · read-only` + : "Read-only conversation" + : "Connecting conversation…", + ), + ); + this.input.addChild(this.readOnly); + } + } + + /** pi: the thinking level's border color, or the bash mode color while the text starts with `!`. */ + private syncBorder(): void { + const theme = this.runtime.theme; + const level = this.current?.agent.thinkingLevel ?? "off"; + const bash = this.participant() !== undefined && this.editor.getText().trimStart().startsWith("!"); + this.editor.borderColor = (text) => + bash ? theme.current.getBashModeBorderColor()(text) : theme.current.getThinkingBorderColor(level)(text); + } + + /** The running command as pi's bash execution component at the end of the transcript. */ + private syncShell(): void { + const live = this.liveShell; + if (!live) { + this.shellView?.component.dispose(); + this.shellView = undefined; + this.transcript.tail = undefined; + return; + } + if (this.shellView?.id !== live.id) { + this.shellView?.component.dispose(); + const component = new BashExecutionComponent(live.command, this.runtime.ctx, { + excludeFromContext: !live.includeInContext, + by: shellAuthor(live.author), + ...(live.running ? { tui: this.runtime.tui } : {}), + }); + this.shellView = { id: live.id, component }; + } + const component = this.shellView.component; + component.setExpanded(this.runtime.expandedTools); + component.setOutput(live.output, live.truncated); + if (!live.running) + component.setComplete(live.exitCode, live.cancelled, { + truncated: live.truncated, + ...(live.error === undefined ? {} : { error: live.error }), + }); + this.transcript.tail = component; + } + + /** `!cmd`/`!!cmd`: runs the command (`Chat.shell`); one at a time, as pi. */ + shell(command: string, includeInContext: boolean): void { + const ref = this.participant(); + const identity = this.identity; + if (!ref || !identity) { + this.runtime.fail( + this.ref?.() !== undefined && statusOf(this.ref()) === "live" + ? "This conversation is read-only" + : "Not connected yet", + ); + return; + } + if (this.liveShell?.running) { + this.runtime.notify(SHELL_BUSY, "warning"); + return; + } + const draft = this.editor.getExpandedText(); + this.runtime.clearNotices(); + this.scroll.scrollToEnd(); + this.runtime.perform( + ref.shell({ command, includeInContext, requestId: randomUUID() }, BACKGROUND_CONTEXT).then(() => { + if (!this.disposed) this.editor.addToHistory(draft.trim()); + if (!this.disposed && identity === this.identity && this.editor.getExpandedText() === draft) + this.editor.setText(""); + if (this.runtime.drafts.get(identity) === draft) this.runtime.drafts.set(identity, ""); + this.runtime.changed(); + }), + ); + } + + /** Queued inputs (coding-agent's pending messages area). */ + private syncPending(participant: boolean): void { + const theme = this.runtime.theme.current; + this.pending.clear(); + const queued = this.current?.queued ?? []; + const replying = this.canReply ? this.replyTo : undefined; + if (queued.length === 0 && !this.sending && !this.waiting && !replying) return; + this.pending.addChild(new Text("", 0, 0)); + if (replying) + this.pending.addChild( + new TruncatedText(theme.fg("accent", `↪ Replying to ${lineText(replying.name)} · /reply cancels`), 1, 0), + ); + for (const input of queued) + this.pending.addChild( + new TruncatedText( + theme.fg("dim", `${input.mode === "followUp" ? "Follow-up" : "Steering"}: ${lineText(input.text)}`), + 1, + 0, + ), + ); + if (queued.length && participant) { + const hint = this.runtime.ctx.keyText("app.message.dequeue"); + if (hint) + this.pending.addChild(new TruncatedText(theme.fg("dim", `↳ ${hint} to edit your queued messages`), 1, 0)); + } + if (this.sending || this.waiting) + this.pending.addChild( + new TruncatedText(theme.fg("dim", this.sending ? "Sending…" : "Sending when connected…"), 1, 0), + ); + } + + /** DTUI `#syncStatus`: retry, compaction, running tool or working, from `pi.live`. */ + private syncIndicator(participant: boolean): void { + const ctx = this.runtime.ctx; + const tui = this.runtime.tui; + const live = this.view?.docs["pi.live"] as unknown as LiveState | undefined; + const generation = live?.generation; + const compaction = live?.compactions?.[0]; + const running = live?.tools?.find((slot) => slot.status === "running"); + const abort = `(${ctx.keyText("app.interrupt")} to abort)`; + let key = ""; + let create: (() => StatusIndicator) | undefined; + if (generation?.retry !== undefined) { + const retry = generation.retry; + key = `retry:${generation.attempt}:${retry.at}`; + create = () => new RetryStatusIndicator(tui, ctx, generation.attempt, retry.at, lineText(retry.error)); + } else if (generation?.deferred !== undefined) { + key = "deferred"; + create = () => new WorkingStatusIndicator(tui, ctx, "Waiting for deferred response..."); + } else if (compaction !== undefined) { + key = `compaction:${compaction.trigger}:${compaction.retry ? compaction.attempt : ""}`; + create = () => + new CompactionStatusIndicator( + tui, + ctx, + compaction.trigger, + compaction.retry ? compaction.attempt : undefined, + ); + } else if (running !== undefined) { + key = `tool:${running.name}`; + create = () => new WorkingStatusIndicator(tui, ctx, `Running ${lineText(running.name)}... ${abort}`); + } else if (live?.run !== undefined) { + key = "working"; + create = () => new WorkingStatusIndicator(tui, ctx, `Working... ${abort}`); + } + key = `${key}:${participant}`; + if (key !== this.indicatorKey) { + this.indicatorKey = key; + this.indicator?.dispose(); + this.indicator = create?.(); + } + this.editor.setWorkingStatusIndicator(participant ? this.indicator : undefined); + this.status.clear(); + // Viewers have no editor border: the indicator gets its own line. + if (!participant && this.indicator) this.status.addChild(this.indicator); + } + + send(mode: "steer" | "followUp"): void { + const ref = this.participant(); + const text = this.editor.getExpandedText().trim(); + const files = this.identity ? (this.attachments.get(this.identity) ?? []) : []; + if ((!text && !files.length) || this.sending) return; + const current = this.ref?.(); + if (current !== undefined && statusOf(current) === "live" && current.variant !== "participant") { + this.runtime.fail("This conversation is read-only"); + return; + } + if (!this.runtime.connected || !ref) { + // Still connecting (a reload, a reconnect): sent once the conversation is live, with the same request ID. + this.waiting = mode; + this.runtime.changed(); + return; + } + this.waiting = undefined; + const identity = this.identity!; + const draft = this.editor.getExpandedText(); + let submission = this.runtime.submissions.get(identity); + if (submission?.text !== text || submission.mode !== mode) { + submission = { text, mode, id: randomUUID() }; + this.runtime.submissions.set(identity, submission); + } + this.sending = true; + this.runtime.clearNotices(); + this.scroll.scrollToEnd(); + this.runtime.perform( + ref + .send( + { + text, + requestId: submission.id, + whenBusy: mode, + ...(files.length ? { attachments: files.map((file) => file.id) } : {}), + ...(this.canReply && this.replyTo ? { replyTo: this.replyTo.entryId } : {}), + }, + BACKGROUND_CONTEXT, + ) + .then(() => { + // Sent: these attachments belong to the message now (later uploads stay pending). + const left = (this.attachments.get(identity) ?? []).filter((file) => !files.includes(file)); + this.attachments.set(identity, left); + if (identity === this.identity) this.replyTo = undefined; + if (!this.disposed) this.editor.addToHistory(text); + if (!this.disposed && identity === this.identity && this.editor.getExpandedText() === draft) + this.editor.setText(""); + if (this.runtime.drafts.get(identity) === draft) this.runtime.drafts.set(identity, ""); + if (this.runtime.submissions.get(identity) === submission) this.runtime.submissions.delete(identity); + }) + .finally(() => { + if (identity === this.identity) this.sending = false; + this.runtime.changed(); + }), + ); + } + + private restore(identity: string, texts: string[]): void { + if (!texts.length) return; + const draft = [...texts, this.runtime.drafts.get(identity) ?? ""].filter(Boolean).join("\n\n"); + this.runtime.drafts.set(identity, draft); + if (!this.disposed && this.identity === identity) this.editor.setText(draft); + this.runtime.changed(); + } + + private loadEarlier(): void { + const ref = this.ref?.(); + const store = this.transcriptStore; + if (ref === undefined || statusOf(ref) !== "live" || !store || this.earlierFailed.has(store)) return; + this.runtime.perform( + (async () => { + try { + await store.loadEarlier(); + } catch (error) { + // No retry loop from the top of the transcript: scrolling there again does not ask again. + this.earlierFailed.add(store); + throw error; + } finally { + this.runtime.changed(); + } + })(), + ); + } + + /** Loads a gap's missing entries while it is in view; a failed load is not retried for that gap. */ + private fillGap(gap: Gap): void { + const ref = this.ref?.(); + const store = this.transcriptStore; + if ( + ref === undefined || + statusOf(ref) !== "live" || + !store || + this.gapsFailed.has(gap) || + store.value?.filling.has(gapKey(gap)) + ) + return; + this.runtime.perform( + (async () => { + try { + await store.fillGap(gap); + } catch (error) { + this.gapsFailed.add(gap); + throw error; + } finally { + this.runtime.changed(); + } + })(), + ); + } + + /** + * `app.interrupt` (pi's Escape): aborts a running compaction and stops the generation; the caller's own queued + * texts come back to the editor. + */ + interrupt(): void { + const ref = this.participant(); + const identity = this.identity; + if (!ref || !identity) return; + const live = this.view?.docs["pi.live"] as unknown as LiveState | undefined; + const shell = this.liveShell; + // pi's order: a running agent, a running command, bash mode's text, then the double Escape on an empty editor. + if (!live?.run && !live?.compactions?.length) { + if (shell?.running) { + this.runtime.perform(ref.abortShell({ shellId: shell.id }, BACKGROUND_CONTEXT)); + return; + } + if (this.editor.getText().trimStart().startsWith("!")) { + this.editor.setText(""); + this.runtime.drafts.set(identity, ""); + this.syncBorder(); + return; + } + if (!this.editor.getText().trim()) this.doubleEscape(); + } + if (live?.compactions?.length) this.runtime.perform(ref.abortCompaction(BACKGROUND_CONTEXT)); + if (live?.run || !live?.compactions?.length) + this.runtime.perform(ref.stop(BACKGROUND_CONTEXT).then(({ restored }) => this.restore(identity, restored))); + } + + /** pi's `doubleEscapeAction`: "fork" opens `/fork`'s picker; "tree" is not supported (no `/tree`), like "none". */ + private doubleEscape(): void { + if (this.runtime.ctx.settings.doubleEscapeAction !== "fork" || this.runtime.role !== "owner") return; + const now = Date.now(); + if (now - this.lastEscape < DOUBLE_ESCAPE_MS) { + this.lastEscape = 0; + this.openFork(); + } else this.lastEscape = now; + } + + /** + * pi's `/fork`: pick a person's message of the loaded transcript; a new session holds what came before it and the + * message waits in its editor (`Sessions.fork`). The main conversation only. + */ + openFork(): void { + const runtime = this.runtime; + const session = this.session; + if (!session) return; + if (this.conversation !== 1) { + runtime.notify("Forking a subagent conversation is not supported yet", "warning"); + return; + } + const messages = (this.current?.items ?? []).flatMap((item) => + item.type === "user" && item.entryId !== undefined && item.text.trim() + ? [{ id: item.entryId, text: item.text }] + : [], + ); + if (!messages.length) { + runtime.notify("No messages to fork from"); + return; + } + const overlay = runtime.ctx.overlay( + "fork", + new UserMessageSelectorComponent( + runtime.ctx, + messages, + (id) => { + overlay.close(); + const text = messages.find((message) => message.id === id)?.text ?? ""; + runtime.perform( + runtime.host.root(async (root) => { + const sessions = await root.get(Sessions, {}); + if (sessions.variant !== "owner") throw new Error("Only the owner forks sessions"); + const created = await sessions.fork({ id: session, conversation: 1, before: id }, root); + runtime.drafts.set(identityKey(created.id, 1), text); + runtime.navigate("chat", created.id, 1); + runtime.notify("Forked to new session"); + }, BACKGROUND_CONTEXT), + ); + }, + () => overlay.close(), + messages.at(-1)?.id, + ), + { width: "100%", anchor: "bottom-center" }, + ); + } + + /** pi's `/clone`: the whole session as a new one (`Sessions.clone`), opened. */ + cloneSession(): void { + const runtime = this.runtime; + const session = this.session; + if (!session) return; + runtime.perform( + runtime.host.root(async (root) => { + const sessions = await root.get(Sessions, {}); + if (sessions.variant !== "owner") throw new Error("Only the owner clones sessions"); + const created = await sessions.clone({ id: session }, root); + runtime.navigate("chat", created.id, 1); + runtime.notify("Cloned to new session"); + }, BACKGROUND_CONTEXT), + ); + } + + /** `/compact [instructions]`: summarize earlier context now (Escape aborts it). */ + compact(instructions?: string): void { + const ref = this.participant(); + if (!ref) return; + this.runtime.perform( + ref.compact(instructions?.trim() ? { instructions: instructions.trim() } : {}, BACKGROUND_CONTEXT), + ); + } + + /** `/new-context [handoff]`: a new context, starting with the handoff text when given (owner). */ + newContext(handoff?: string): void { + const ref = this.participant(); + if (!ref) return; + this.runtime.perform( + ref + .reset(handoff?.trim() ? { handoff: handoff.trim() } : {}, BACKGROUND_CONTEXT) + .then(() => this.runtime.notify("Started a new context")), + ); + } + + /** `app.clear` (pi's Ctrl+C): clears the editor; a second press within 500 ms exits. */ + clear(): void { + const now = Date.now(); + if (now - this.lastCtrlC < DOUBLE_CTRL_C_MS) { + this.lastCtrlC = 0; + this.runtime.onExit(); + return; + } + this.lastCtrlC = now; + this.editor.setText(""); + if (this.identity) this.runtime.drafts.set(this.identity, ""); + } + + /** + * `app.message.dequeue` (pi's Alt+Up): every queued input of the caller back to the editor, in order, before the + * current text; other people's stay queued (`Chat.dequeueOwn`). + */ + dequeueOwn(): void { + const ref = this.participant(); + const identity = this.identity; + if (!ref || !identity) return; + this.runtime.perform( + ref.dequeueOwn(BACKGROUND_CONTEXT).then(({ restored, skipped }) => { + this.restore(identity, restored); + if (skipped) + this.runtime.notify( + `${skipped} queued message${skipped === 1 ? " was" : "s were"} already sent`, + "warning", + ); + else if (!restored.length) this.runtime.notify("No queued messages of yours"); + }), + ); + } + + /** Chat mode switched replies on or off (chat-mode's composer chip). */ + setCanReply(enabled: boolean, mentionName = "Pi"): void { + if (enabled === this.canReply && mentionName === this.mentionName) return; + this.canReply = enabled; + this.mentionName = mentionName; + if (!enabled) this.replyTo = undefined; + this.runtime.changed(); + } + + /** The next send answers this entry (chat mode); undefined cancels. */ + setReplyTo(target: { entryId: number; name: string } | undefined): void { + this.replyTo = target; + this.runtime.changed(); + } + + /** Messages a reply can answer: people's and Pi's committed messages, newest first. */ + replyTargets(): ReplyTarget[] { + return (this.current?.items ?? []) + .flatMap((item) => + item.type === "user" && item.entryId !== undefined + ? [{ entryId: item.entryId, name: item.author?.name ?? item.peer?.title ?? "You", excerpt: item.text }] + : item.type === "assistant" && item.entryId !== undefined + ? [ + { + entryId: item.entryId, + name: "Pi", + excerpt: item.message.content + .flatMap((part) => (part.type === "text" ? [part.text] : [])) + .join(" "), + }, + ] + : [], + ) + .toReversed() + .slice(0, 50); + } + + /** Context terms for command `when` expressions (section 6.3). */ + *contexts(): Iterable { + if (this.current?.working) yield "busy"; + if (this.current?.compactions.length) yield "compacting"; + if (!this.participant()) return; + yield "participant"; + yield "editorFocused"; + if (!this.editor.getText()) yield "editorEmpty"; + if (this.editor.isShowingAutocomplete()) yield "autocomplete"; + if (this.canReply) yield "canReply"; + if (this.identity && this.attachments.get(this.identity)?.length) yield "attachments"; + } + + /** The editor keeps input while the conversation connects (a reload, a reconnect); read-only views drop it. */ + handleInput(data: string): void { + const ref = this.ref?.(); + if (ref !== undefined && statusOf(ref) === "live" && ref.variant !== "participant") return; + this.editor.handleInput(data); + } + + /** Without the fullscreen layout (tests): transcript, dock. */ + render(width: number): string[] { + return [...this.transcript.render(width), ...this.dock.render(width)]; + } + + invalidate(): void { + this.transcript.invalidate(); + this.editor.invalidate(); + } + + dispose(): void { + this.disposed = true; + this.saveState(); + for (const stop of this.stops.splice(0)) stop(); + this.indicator?.dispose(); + this.shellView?.component.dispose(); + this.transcript.dispose(); + this.sessions.dispose(); + this.scope?.dispose(); + } +} + +// The chat plugin defines the transcript's entry displays (chat-mode adds its own): served on the TUI too. +export default tuiFacet( + "chat", + (runtime) => new ChatSurface(runtime), + CHAT_COMMANDS, + (host) => serveKeyed(host, TranscriptEntries), +); diff --git a/apps/p2/plugins/chat/web.tsx b/apps/p2/plugins/chat/web.tsx new file mode 100644 index 000000000..0fb5bf9fb --- /dev/null +++ b/apps/p2/plugins/chat/web.tsx @@ -0,0 +1,1384 @@ +// Main view: the transcript of the open session with its composer, or the new-session screen. + +import { + Access, + BACKGROUND_CONTEXT, + defineFacet, + expandComposerText, + grantMode, + type Held, + isChordError, + liveIn, + serveKeyed, + whenLive, +} from "@p2/app/v1"; +import { + AttachIcon, + buildTranscript, + ChevronIcon, + CloseIcon, + CostPanel, + closeLayers, + conversationRoute, + DownIcon, + type FileCompletion, + FileCompletionPopover, + FillGap, + FolderIcon, + forgetSession, + type Gap, + type Kept, + keep, + keepSessions, + keptSizes, + loadKept, + locationLabel, + MenuIcon, + Messages, + type ModelChoice, + ModelPicker, + modelName, + navigate, + OneShotAgents, + OpenConversation, + OpenWorkspaceFile, + openLayer, + type PickedFile, + PiLogo, + PlusIcon, + ReplyTo, + RevealEntry, + revealEntry, + revealRequest, + rowEstimates, + SendIcon, + SessionId, + SessionStats, + Slot, + SlotList, + Slots, + StopIcon, + seedSizes, + setDrawer, + setSidebarCollapsed, + takeReveal, + transcriptOf, + transcriptRows, + useChatScroll, + useComposerState, + useConnected, + useFileCompletion, + useHost, + useNavigation, + useNewSessionState, + useObject, + useOwner, + useReadable, + useRegistry, + useRoute, + useShareEnded, + useSheet, + useStatus, + WorkingDirectory, +} from "@p2/app/v1/web"; +import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, useSyncExternalStore } from "react"; +import { Drafts } from "../drafts/contract.ts"; +import { DirectoryPicker, directoryPickerLayer } from "../envs/contract.web.tsx"; +import { Models } from "../models/contract.ts"; +import type { ResourceSnapshot } from "../models/resources.ts"; +import { type Session, Sessions } from "../sessions/contract.ts"; +import { SessionMetrics, SessionStatus, WorkerControl } from "../status/contract.ts"; +import { type Child, conversationLocation, SubagentControl, Subagents } from "../subagents/contract.ts"; +import { Chat, ChatCosts, TranscriptEntries } from "./contract.ts"; +import { type Outgoing, outboxOf } from "./outbox.ts"; + +const errorText = (error: unknown): string => (error instanceof Error ? error.message : String(error)); + +/** An upload the server answered with an error status. */ +class UploadError extends Error { + readonly status: number; + constructor(message: string, status: number) { + super(message); + this.status = status; + } +} + +/** Uploads a file to the session (the files plugin's route); returns its upload ID. */ +async function upload(session: string, file: File): Promise { + const query = new URLSearchParams({ session, name: file.name }); + const response = await fetch(`/upload?${query}`, { method: "POST", body: file, credentials: "same-origin" }); + const result = (await response.json().catch(() => ({}))) as { id?: string; error?: string }; + if (!response.ok || !result.id) + throw new UploadError(`Upload of ${file.name} failed: ${result.error ?? response.status}`, response.status); + return result.id; +} + +/** + * Whether a failed send refuses the message for good: not allowed, invalid (a template that does not exist), an upload + * the server rejects. Anything else (the connection, a restarting worker) is retried by the outbox. + */ +function refused(error: unknown): boolean { + if (error instanceof UploadError) return error.status >= 400 && error.status < 500; + return ( + isChordError(error) && + ["forbidden", "invalid", "not_in_variant", "contract_mismatch", "upgrade_required"].includes(error.code) + ); +} + +/** Picked files: shown above the textarea, removable; the attach button adds more. */ +type Attaching = { + readonly files: readonly PickedFile[]; + readonly add: (files: File[]) => void; + readonly remove: (file: PickedFile) => void; +}; + +/** The composer's attachments from its state. */ +const attachingOf = ({ files, setFiles, addFiles }: ReturnType): Attaching => ({ + files, + add: addFiles, + remove: (file) => setFiles((current) => current.filter((entry) => entry !== file)), +}); + +/** The model and thinking chip that opens the model sheet. */ +function ModelControl({ resources, choice }: { resources: ResourceSnapshot | undefined; choice?: ModelChoice }) { + const name = choice ? modelName(resources, choice.model) : "Choose model"; + const level = choice && choice.thinkingLevel !== "off" ? choice.thinkingLevel : undefined; + return ( + + ); +} + +/** The composer's error; tapping it dismisses it. */ +function ErrorToast({ error, onDismiss }: { error: string | undefined; onDismiss: () => void }) { + return error ? ( + + ) : null; +} + +/** Textarea with send, stop, steer and attachments; Enter sends with a fine pointer, Escape stops. */ +function Composer({ + draft, + setDraft, + busy, + steer, + setSteer, + disabled, + onSend, + onStop, + attaching, + replying, + completion, + textareaRef, + children, +}: { + draft: string; + setDraft: (text: string) => void; + busy: boolean; + steer: boolean; + setSteer: (steer: boolean) => void; + disabled: boolean; + onSend: () => void; + onStop?: () => void; + attaching?: Attaching; + /** The message being answered, shown above the text with a way to drop the reply. */ + replying?: { name: string; onCancel: () => void }; + /** `@` and Tab file completion (p2 addition, src/web/file-completion.tsx); only where a session exists. */ + completion?: FileCompletion; + textareaRef?: React.RefObject; + children?: React.ReactNode; +}) { + const ownRef = useRef(null); + const textarea = textareaRef ?? ownRef; + // biome-ignore lint/correctness/useExhaustiveDependencies: resizing follows the rendered draft, as in Pi Mobile. + useLayoutEffect(() => { + const element = textarea.current; + if (!element) return; + element.style.height = "auto"; + element.style.overflowY = "hidden"; + element.style.height = `${Math.min(element.scrollHeight + 1, 200)}px`; + element.style.overflowY = element.scrollHeight > element.clientHeight ? "auto" : "hidden"; + }, [draft]); + const files = attaching?.files ?? []; + const content = draft.trim() !== "" || files.length > 0; + return ( +
{ + event.preventDefault(); + onSend(); + }} + > + {replying && ( +
+ + Replying to {replying.name} + + +
+ )} + {files.length > 0 && ( +
+ {files.map((file) => ( + + {file.preview ? : } + {file.file.name} + + + ))} +
+ )} + {completion && } +